{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-16 08:37:54.448454: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-11-16 08:37:54.663063: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-11-16 08:37:54.664267: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-11-16 08:37:56.555501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "/home/dell/anaconda3/envs/CX/lib/python3.8/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import gym\n",
    "import imageio\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow_probability as tfp\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm as tool_bar\n",
    "import scipy.signal\n",
    "import time\n",
    "import imageio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PPO_Buffer:\n",
    "    def __init__(self,dimension_state,dimension_action,size=2048,gamma=0.99,lam=0.95):\n",
    "        self.buffer_state = np.zeros((size,dimension_state),dtype=np.float32)\n",
    "        self.buffer_action = np.zeros((size,dimension_action),dtype=np.float32)\n",
    "        self.buffer_reward = np.zeros(size,dtype=np.float32)\n",
    "        self.buffer_value = np.zeros(size,dtype=np.float32)\n",
    "        self.buffer_return = np.zeros(size,dtype=np.float32)\n",
    "        self.buffer_advantage = np.zeros(size,dtype=np.float32)\n",
    "        self.buffer_log_probability = np.zeros((size,dimension_action),dtype=np.float32)\n",
    "        self.gamma = gamma\n",
    "        self.lam = lam\n",
    "        self.pointer = 0\n",
    "        self.trajectory_start_index = 0\n",
    "    def discounted_cumulative_sums(self,x,discount):\n",
    "        return scipy.signal.lfilter([1],[1,float(-discount)],x[::-1],axis=0)[::-1]\n",
    "    def store(self,state,action,reward,value,log_probability):\n",
    "        self.buffer_state[self.pointer] = state\n",
    "        self.buffer_action[self.pointer] = action\n",
    "        self.buffer_reward[self.pointer] = reward\n",
    "        self.buffer_value[self.pointer] = value\n",
    "        self.buffer_log_probability[self.pointer] = log_probability\n",
    "        self.pointer += 1\n",
    "    def finish_trajectory(self,last_value=0):\n",
    "        path_slice = slice(self.trajectory_start_index,self.pointer)\n",
    "        rewards = np.append(self.buffer_reward[path_slice],last_value)\n",
    "        values = np.append(self.buffer_value[path_slice],last_value)\n",
    "        deltas = rewards[:-1] + self.gamma*values[1:]-values[:-1]\n",
    "        self.buffer_return[path_slice] = self.discounted_cumulative_sums(rewards,self.gamma)[:-1]\n",
    "        self.buffer_advantage[path_slice] = self.discounted_cumulative_sums(deltas,self.gamma*self.lam)\n",
    "        self.trajectory_start_index = self.pointer\n",
    "    def get(self):\n",
    "        self.pointer = 0\n",
    "        self.trajectory_start_index = 0\n",
    "        advantage_mean = np.mean(self.buffer_advantage)\n",
    "        advantage_std = np.std(self.buffer_advantage)\n",
    "        self.buffer_advantage = (self.buffer_advantage-advantage_mean)/advantage_std\n",
    "        return [self.buffer_state,self.buffer_action,self.buffer_advantage,self.buffer_return,self.buffer_log_probability]\n",
    "class PPO_Policy_Model(tf.keras.Model):\n",
    "    def __init__(self,dimension_action,layer_dimension=[256,512],activation='relu',scale=1):\n",
    "        super(PPO_Policy_Model,self).__init__()\n",
    "        self.scale = scale\n",
    "        self.layer = [tf.keras.layers.Dense(i,activation=activation) for i in layer_dimension]\n",
    "        self.policy_mean = tf.keras.layers.Dense(dimension_action,activation='tanh')\n",
    "        self.policy_standard_deviation = tf.keras.layers.Dense(dimension_action,activation='sigmoid')\n",
    "    def call(self,x):\n",
    "        y = x*1\n",
    "        for i in self.layer:\n",
    "            y = i(y)\n",
    "        policy_mean = self.policy_mean(y)\n",
    "        policy_standard_deviation = self.policy_standard_deviation(y)\n",
    "        return [policy_mean*self.scale*0.8,policy_standard_deviation*self.scale/10+0.1]\n",
    "class PPO_Value_Model(tf.keras.Model):\n",
    "    def __init__(self,dimension_action,layer_dimension=[256,512],activation='relu'):\n",
    "        super(PPO_Value_Model,self).__init__()\n",
    "        self.layer = [tf.keras.layers.Dense(i,activation=activation) for i in layer_dimension]\n",
    "        self.value = tf.keras.layers.Dense(1,activation='linear')\n",
    "    def call(self,x):\n",
    "        y = x*1\n",
    "        for i in self.layer:\n",
    "            y = i(y)\n",
    "        value = self.value(y)\n",
    "        return value\n",
    "class PPO_Agent:\n",
    "    def __init__(self,environment_name):\n",
    "        self.instance_environment = gym.make(environment_name) \n",
    "        self.dimension_action = self.instance_environment.action_space.shape[0]\n",
    "        self.dimension_state = self.instance_environment.observation_space.shape[0]\n",
    "        self.action_value_bound = np.array([self.instance_environment.action_space.low[0],self.instance_environment.action_space.high[0]])\n",
    "        self.action_value_scale = np.max(abs(self.action_value_bound))\n",
    "        self.gamma = 0.99\n",
    "        self.epsilon = 0.2\n",
    "        self.steps_per_epoch = 1024*4\n",
    "        self.policy_learning_rate = 3e-4\n",
    "        self.value_function_learning_rate = 1e-3\n",
    "        self.lam = 0.97\n",
    "        self.target_kl = 0.01\n",
    "        self.training_iteration = [20,200]\n",
    "        self.instance_PPO_Policy_Model = PPO_Policy_Model(self.dimension_action,scale=self.action_value_scale)\n",
    "        self.instance_PPO_Value_Model = PPO_Value_Model(self.dimension_action)\n",
    "        self.instance_PPO_Policy_Model.compile(optimizer=tf.keras.optimizers.Adam(self.policy_learning_rate))\n",
    "        self.instance_PPO_Value_Model.compile(optimizer=tf.keras.optimizers.Adam(self.value_function_learning_rate))\n",
    "        _ = self.instance_PPO_Policy_Model(tf.random.normal(shape=[1,self.dimension_state]))\n",
    "        _ = self.instance_PPO_Value_Model(tf.random.normal(shape=[1,self.dimension_state]))\n",
    "        self.instance_PPO_Buffer = PPO_Buffer(self.dimension_state,self.dimension_action,self.steps_per_epoch,gamma=self.gamma,lam=self.lam)\n",
    "        self.history_training = [[],[],[]]\n",
    "    def save_model(self,save_path='./'):\n",
    "        self.instance_PPO_Policy_Model.save_weights(save_path+'policy_model_weight.h5')\n",
    "        self.instance_PPO_Value_Model.save_weights(save_path+'value_model_weight.h5')\n",
    "    def load_model(self,load_path='./'):\n",
    "        self.instance_PPO_Policy_Model.load_weights(load_path+'policy_model_weight.h5')\n",
    "        self.instance_PPO_Value_Model.load_weights(load_path+'value_model_weight.h5')\n",
    "    def get_action_and_distribution(self,state):\n",
    "        state = tf.convert_to_tensor([state])\n",
    "        policy_mean,policy_standard_deviation = self.instance_PPO_Policy_Model(state)\n",
    "        distribution_action = tfp.distributions.Normal(policy_mean[0],policy_standard_deviation[0])\n",
    "        action = tf.squeeze(distribution_action.sample(1),axis=0)\n",
    "        return [action.numpy(),distribution_action]\n",
    "    def get_log_probability(self,distribution_action,action):\n",
    "        log_probability = tf.reduce_mean(distribution_action.log_prob(action))\n",
    "        return log_probability\n",
    "    #@tf.function\n",
    "    def train_policy(self,buffer_state,buffer_action,buffer_advantage,buffer_log_probability):\n",
    "        with tf.GradientTape() as tape:\n",
    "            policy_mean,policy_standard_deviation = self.instance_PPO_Policy_Model(buffer_state)\n",
    "            distribution_action = tfp.distributions.Normal(policy_mean,policy_standard_deviation)\n",
    "            log_probability = tf.reduce_mean(distribution_action.log_prob(buffer_action),axis=-1)\n",
    "            log_probability_old = tf.reduce_mean(buffer_log_probability,axis=-1)\n",
    "            ratio = tf.exp(log_probability-log_probability_old)\n",
    "            min_advantage = tf.where(buffer_advantage>0,(1+self.epsilon)*buffer_advantage,(1-self.epsilon)*buffer_advantage)\n",
    "            loss_policy = -tf.reduce_mean(tf.minimum(ratio*buffer_advantage,min_advantage))\n",
    "        grad_policy = tape.gradient(loss_policy,self.instance_PPO_Policy_Model.trainable_variables)\n",
    "        self.instance_PPO_Policy_Model.optimizer.apply_gradients(zip(grad_policy,self.instance_PPO_Policy_Model.trainable_variables))\n",
    "        kl = tf.reduce_mean(log_probability-log_probability_old)\n",
    "        kl = tf.reduce_sum(kl)\n",
    "        self.history_training[0].append(loss_policy)\n",
    "        self.history_training[0] = self.history_training[0][-1000:]\n",
    "        return kl\n",
    "    #@tf.function\n",
    "    def train_value_function(self,buffer_state,buffer_return):\n",
    "        with tf.GradientTape() as tape:\n",
    "            value = tf.squeeze(self.instance_PPO_Value_Model(buffer_state),axis=-1)\n",
    "            loss_value = tf.reduce_mean((buffer_return-value)**2)\n",
    "        grad_value = tape.gradient(loss_value,self.instance_PPO_Value_Model.trainable_variables)\n",
    "        self.instance_PPO_Value_Model.optimizer.apply_gradients(zip(grad_value,self.instance_PPO_Value_Model.trainable_variables))\n",
    "        self.history_training[1].append(loss_value)\n",
    "        self.history_training[1] = self.history_training[1][-1000:]\n",
    "    def explore(self,epochs=20):\n",
    "        episode_return = 0\n",
    "        episode_length = 0\n",
    "        for epoch in range(epochs):\n",
    "            sum_return = 0\n",
    "            sum_length = 0\n",
    "            num_episodes = 0\n",
    "            state = self.instance_environment.reset()\n",
    "            for t in range(self.steps_per_epoch):\n",
    "                state = state.reshape(1,-1)\n",
    "                action,action_distribution = self.get_action_and_distribution(state)\n",
    "                value = self.instance_PPO_Value_Model(state)\n",
    "                log_probability = self.get_log_probability(action_distribution,action)\n",
    "                state_next,reward,done,info = self.instance_environment.step(action)\n",
    "                reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']+0.0005*info['reward_alive']\n",
    "                episode_return += reward\n",
    "                episode_length += 1\n",
    "                self.instance_PPO_Buffer.store(state,action,reward,value,log_probability)\n",
    "                state = state_next\n",
    "                terminal = done\n",
    "                if terminal or (t==self.steps_per_epoch-1):\n",
    "                    last_value = 0 if done else self.instance_PPO_Value_Model(state.reshape(1,-1))[0]\n",
    "                    self.instance_PPO_Buffer.finish_trajectory(last_value)\n",
    "                    sum_return += episode_return\n",
    "                    sum_length += episode_length\n",
    "                    num_episodes += 1\n",
    "                    state = self.instance_environment.reset()\n",
    "                    episode_return = 0\n",
    "                    episode_length = 0\n",
    "            buffer_state,buffer_action,buffer_advantage,buffer_return,buffer_log_probability = self.instance_PPO_Buffer.get()\n",
    "            for _ in range(self.training_iteration[0]):\n",
    "                kl = self.train_policy(buffer_state,buffer_action,buffer_advantage,buffer_log_probability)\n",
    "                if kl > 1.5*self.target_kl:\n",
    "                    break\n",
    "            for _ in range(self.training_iteration[1]):\n",
    "                self.train_value_function(buffer_state,buffer_return)\n",
    "            self.history_training[2].append(sum_return/num_episodes)\n",
    "            self.history_training[2] = self.history_training[2][-1000:]\n",
    "            print(\n",
    "                f\" Epoch: {epoch + 1}. Mean Return: {sum_return/num_episodes}. Mean Length: {sum_length/num_episodes}\"\n",
    "            )\n",
    "    def test(self):\n",
    "        video = []\n",
    "        state = self.instance_environment.reset()\n",
    "        episodic_reward = 0\n",
    "        while True:\n",
    "            state = state.reshape(1,-1)\n",
    "            action,action_distribution = self.get_action_and_distribution(state)\n",
    "            state_next,reward,done,info = self.instance_environment.step(action)\n",
    "            reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']\n",
    "            episodic_reward += reward\n",
    "            state = state_next\n",
    "            video.append(self.instance_environment.render(mode ='rgb_array'))\n",
    "            if done:\n",
    "                break\n",
    "        writer = imageio.get_writer('test_video.mp4',fps=30) \n",
    "        for image in video:\n",
    "            writer.append_data(np.uint8(image))\n",
    "        writer.close()\n",
    "        print('Video saved to test_video.mp4 with reward '+str(episodic_reward)+'.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_PPO_Agent = PPO_Agent('Humanoid-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_PPO_Agent.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_episode = 740"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch: 1. Mean Return: 32.128295536999666. Mean Length: 56.888888888888886\n",
      " Epoch: 2. Mean Return: 33.409932140638084. Mean Length: 57.690140845070424\n",
      " Epoch: 3. Mean Return: 34.198949539764385. Mean Length: 60.23529411764706\n",
      " Epoch: 4. Mean Return: 35.21067525439043. Mean Length: 58.51428571428571\n",
      " Epoch: 5. Mean Return: 34.85160280777158. Mean Length: 60.23529411764706\n",
      " Epoch: 6. Mean Return: 31.757272592398063. Mean Length: 56.10958904109589\n",
      " Epoch: 7. Mean Return: 36.87658694816984. Mean Length: 59.36231884057971\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 7\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m epochs \u001b[39m=\u001b[39m \u001b[39m20\u001b[39m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m instance_PPO_Agent\u001b[39m.\u001b[39;49mexplore(epochs)\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m plt\u001b[39m.\u001b[39mplot(instance_PPO_Agent\u001b[39m.\u001b[39mhistory_training[\u001b[39m2\u001b[39m])\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m plt\u001b[39m.\u001b[39mshow()\n",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 7\u001b[0m in \u001b[0;36mPPO_Agent.explore\u001b[0;34m(self, epochs)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=134'>135</a>\u001b[0m \u001b[39mfor\u001b[39;00m t \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msteps_per_epoch):\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=135'>136</a>\u001b[0m     state \u001b[39m=\u001b[39m state\u001b[39m.\u001b[39mreshape(\u001b[39m1\u001b[39m,\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m--> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=136'>137</a>\u001b[0m     action,action_distribution \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_action_and_distribution(state)\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=137'>138</a>\u001b[0m     value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minstance_PPO_Value_Model(state)\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=138'>139</a>\u001b[0m     log_probability \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_log_probability(action_distribution,action)\n",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 7\u001b[0m in \u001b[0;36mPPO_Agent.get_action_and_distribution\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=92'>93</a>\u001b[0m state \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39mconvert_to_tensor([state])\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=93'>94</a>\u001b[0m policy_mean,policy_standard_deviation \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minstance_PPO_Policy_Model(state)\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=94'>95</a>\u001b[0m distribution_action \u001b[39m=\u001b[39m tfp\u001b[39m.\u001b[39;49mdistributions\u001b[39m.\u001b[39;49mNormal(policy_mean[\u001b[39m0\u001b[39;49m],policy_standard_deviation[\u001b[39m0\u001b[39;49m])\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=95'>96</a>\u001b[0m action \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39msqueeze(distribution_action\u001b[39m.\u001b[39msample(\u001b[39m1\u001b[39m),axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W3sZmlsZQ%3D%3D?line=96'>97</a>\u001b[0m \u001b[39mreturn\u001b[39;00m [action\u001b[39m.\u001b[39mnumpy(),distribution_action]\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/decorator.py:232\u001b[0m, in \u001b[0;36mdecorate.<locals>.fun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m    230\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m kwsyntax:\n\u001b[1;32m    231\u001b[0m     args, kw \u001b[39m=\u001b[39m fix(args, kw, sig)\n\u001b[0;32m--> 232\u001b[0m \u001b[39mreturn\u001b[39;00m caller(func, \u001b[39m*\u001b[39;49m(extras \u001b[39m+\u001b[39;49m args), \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkw)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/distributions/distribution.py:342\u001b[0m, in \u001b[0;36m_DistributionMeta.__new__.<locals>.wrapped_init\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    339\u001b[0m \u001b[39m# Note: if we ever want to have things set in `self` before `__init__` is\u001b[39;00m\n\u001b[1;32m    340\u001b[0m \u001b[39m# called, here is the place to do it.\u001b[39;00m\n\u001b[1;32m    341\u001b[0m self_\u001b[39m.\u001b[39m_parameters \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 342\u001b[0m default_init(self_, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    343\u001b[0m \u001b[39m# Note: if we ever want to override things set in `self` by subclass\u001b[39;00m\n\u001b[1;32m    344\u001b[0m \u001b[39m# `__init__`, here is the place to do it.\u001b[39;00m\n\u001b[1;32m    345\u001b[0m \u001b[39mif\u001b[39;00m self_\u001b[39m.\u001b[39m_parameters \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    346\u001b[0m   \u001b[39m# We prefer subclasses will set `parameters = dict(locals())` because\u001b[39;00m\n\u001b[1;32m    347\u001b[0m   \u001b[39m# this has nearly zero overhead. However, failing to do this, we will\u001b[39;00m\n\u001b[1;32m    348\u001b[0m   \u001b[39m# resolve the input arguments dynamically and only when needed.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/distributions/normal.py:135\u001b[0m, in \u001b[0;36mNormal.__init__\u001b[0;34m(self, loc, scale, validate_args, allow_nan_stats, name)\u001b[0m\n\u001b[1;32m    133\u001b[0m parameters \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m(\u001b[39mlocals\u001b[39m())\n\u001b[1;32m    134\u001b[0m \u001b[39mwith\u001b[39;00m tf\u001b[39m.\u001b[39mname_scope(name) \u001b[39mas\u001b[39;00m name:\n\u001b[0;32m--> 135\u001b[0m   dtype \u001b[39m=\u001b[39m dtype_util\u001b[39m.\u001b[39;49mcommon_dtype([loc, scale], dtype_hint\u001b[39m=\u001b[39;49mtf\u001b[39m.\u001b[39;49mfloat32)\n\u001b[1;32m    136\u001b[0m   \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_loc \u001b[39m=\u001b[39m tensor_util\u001b[39m.\u001b[39mconvert_nonref_to_tensor(\n\u001b[1;32m    137\u001b[0m       loc, dtype\u001b[39m=\u001b[39mdtype, name\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mloc\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m    138\u001b[0m   \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_scale \u001b[39m=\u001b[39m tensor_util\u001b[39m.\u001b[39mconvert_nonref_to_tensor(\n\u001b[1;32m    139\u001b[0m       scale, dtype\u001b[39m=\u001b[39mdtype, name\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mscale\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/internal/dtype_util.py:230\u001b[0m, in \u001b[0;36mcommon_dtype\u001b[0;34m(args, dtype_hint)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39mif\u001b[39;00m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mis_nested(dtype) \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mis_nested(dtype_hint):\n\u001b[1;32m    229\u001b[0m   dtype_hint \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mmap_structure(\u001b[39mlambda\u001b[39;00m _: dtype_hint, dtype)\n\u001b[0;32m--> 230\u001b[0m \u001b[39mreturn\u001b[39;00m tf\u001b[39m.\u001b[39;49mnest\u001b[39m.\u001b[39;49mmap_structure(\n\u001b[1;32m    231\u001b[0m     \u001b[39mlambda\u001b[39;49;00m dt, h: base_dtype(h \u001b[39mif\u001b[39;49;00m dt \u001b[39mis\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m \u001b[39melse\u001b[39;49;00m dt), dtype, dtype_hint)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/nest.py:624\u001b[0m, in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    538\u001b[0m \u001b[39m@tf_export\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mnest.map_structure\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    539\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmap_structure\u001b[39m(func, \u001b[39m*\u001b[39mstructure, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    540\u001b[0m \u001b[39m  \u001b[39m\u001b[39m\"\"\"Creates a new structure by applying `func` to each atom in `structure`.\u001b[39;00m\n\u001b[1;32m    541\u001b[0m \n\u001b[1;32m    542\u001b[0m \u001b[39m  Refer to [tf.nest](https://www.tensorflow.org/api_docs/python/tf/nest)\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    622\u001b[0m \u001b[39m    ValueError: If wrong keyword arguments are provided.\u001b[39;00m\n\u001b[1;32m    623\u001b[0m \u001b[39m  \"\"\"\u001b[39;00m\n\u001b[0;32m--> 624\u001b[0m   \u001b[39mreturn\u001b[39;00m nest_util\u001b[39m.\u001b[39;49mmap_structure(\n\u001b[1;32m    625\u001b[0m       nest_util\u001b[39m.\u001b[39;49mModality\u001b[39m.\u001b[39;49mCORE, func, \u001b[39m*\u001b[39;49mstructure, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[1;32m    626\u001b[0m   )\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/nest_util.py:1054\u001b[0m, in \u001b[0;36mmap_structure\u001b[0;34m(modality, func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    957\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Creates a new structure by applying `func` to each atom in `structure`.\u001b[39;00m\n\u001b[1;32m    958\u001b[0m \n\u001b[1;32m    959\u001b[0m \u001b[39m- For Modality.CORE: Refer to\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1051\u001b[0m \u001b[39m  ValueError: If wrong keyword arguments are provided.\u001b[39;00m\n\u001b[1;32m   1052\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   1053\u001b[0m \u001b[39mif\u001b[39;00m modality \u001b[39m==\u001b[39m Modality\u001b[39m.\u001b[39mCORE:\n\u001b[0;32m-> 1054\u001b[0m   \u001b[39mreturn\u001b[39;00m _tf_core_map_structure(func, \u001b[39m*\u001b[39;49mstructure, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1055\u001b[0m \u001b[39melif\u001b[39;00m modality \u001b[39m==\u001b[39m Modality\u001b[39m.\u001b[39mDATA:\n\u001b[1;32m   1056\u001b[0m   \u001b[39mreturn\u001b[39;00m _tf_data_map_structure(func, \u001b[39m*\u001b[39mstructure, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/nest_util.py:1094\u001b[0m, in \u001b[0;36m_tf_core_map_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m   1089\u001b[0m flat_structure \u001b[39m=\u001b[39m (_tf_core_flatten(s, expand_composites) \u001b[39mfor\u001b[39;00m s \u001b[39min\u001b[39;00m structure)\n\u001b[1;32m   1090\u001b[0m entries \u001b[39m=\u001b[39m \u001b[39mzip\u001b[39m(\u001b[39m*\u001b[39mflat_structure)\n\u001b[1;32m   1092\u001b[0m \u001b[39mreturn\u001b[39;00m _tf_core_pack_sequence_as(\n\u001b[1;32m   1093\u001b[0m     structure[\u001b[39m0\u001b[39m],\n\u001b[0;32m-> 1094\u001b[0m     [func(\u001b[39m*\u001b[39mx) \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m entries],\n\u001b[1;32m   1095\u001b[0m     expand_composites\u001b[39m=\u001b[39mexpand_composites,\n\u001b[1;32m   1096\u001b[0m )\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/nest_util.py:1094\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1089\u001b[0m flat_structure \u001b[39m=\u001b[39m (_tf_core_flatten(s, expand_composites) \u001b[39mfor\u001b[39;00m s \u001b[39min\u001b[39;00m structure)\n\u001b[1;32m   1090\u001b[0m entries \u001b[39m=\u001b[39m \u001b[39mzip\u001b[39m(\u001b[39m*\u001b[39mflat_structure)\n\u001b[1;32m   1092\u001b[0m \u001b[39mreturn\u001b[39;00m _tf_core_pack_sequence_as(\n\u001b[1;32m   1093\u001b[0m     structure[\u001b[39m0\u001b[39m],\n\u001b[0;32m-> 1094\u001b[0m     [func(\u001b[39m*\u001b[39;49mx) \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m entries],\n\u001b[1;32m   1095\u001b[0m     expand_composites\u001b[39m=\u001b[39mexpand_composites,\n\u001b[1;32m   1096\u001b[0m )\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/internal/dtype_util.py:231\u001b[0m, in \u001b[0;36mcommon_dtype.<locals>.<lambda>\u001b[0;34m(dt, h)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39mif\u001b[39;00m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mis_nested(dtype) \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mis_nested(dtype_hint):\n\u001b[1;32m    229\u001b[0m   dtype_hint \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mmap_structure(\u001b[39mlambda\u001b[39;00m _: dtype_hint, dtype)\n\u001b[1;32m    230\u001b[0m \u001b[39mreturn\u001b[39;00m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mmap_structure(\n\u001b[0;32m--> 231\u001b[0m     \u001b[39mlambda\u001b[39;00m dt, h: base_dtype(h \u001b[39mif\u001b[39;49;00m dt \u001b[39mis\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m \u001b[39melse\u001b[39;49;00m dt), dtype, dtype_hint)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/internal/dtype_util.py:75\u001b[0m, in \u001b[0;36mbase_dtype\u001b[0;34m(dtype)\u001b[0m\n\u001b[1;32m     73\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mbase_dtype\u001b[39m(dtype):\n\u001b[1;32m     74\u001b[0m \u001b[39m  \u001b[39m\u001b[39m\"\"\"Returns a non-reference `dtype` based on this `dtype`.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 75\u001b[0m   dtype \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39mif\u001b[39;00m dtype \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m tf\u001b[39m.\u001b[39;49mas_dtype(dtype)\n\u001b[1;32m     76\u001b[0m   \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(dtype, \u001b[39m'\u001b[39m\u001b[39mbase_dtype\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[1;32m     77\u001b[0m     \u001b[39mreturn\u001b[39;00m dtype\u001b[39m.\u001b[39mbase_dtype\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:819\u001b[0m, in \u001b[0;36mas_dtype\u001b[0;34m(type_value)\u001b[0m\n\u001b[1;32m    787\u001b[0m \u001b[39m@tf_export\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mdtypes.as_dtype\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mas_dtype\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    788\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mas_dtype\u001b[39m(type_value):\n\u001b[1;32m    789\u001b[0m \u001b[39m  \u001b[39m\u001b[39m\"\"\"Converts the given `type_value` to a `tf.DType`.\u001b[39;00m\n\u001b[1;32m    790\u001b[0m \n\u001b[1;32m    791\u001b[0m \u001b[39m  Inputs can be existing `tf.DType` objects, a [`DataType`\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    817\u001b[0m \u001b[39m    TypeError: If `type_value` cannot be converted to a `DType`.\u001b[39;00m\n\u001b[1;32m    818\u001b[0m \u001b[39m  \"\"\"\u001b[39;00m\n\u001b[0;32m--> 819\u001b[0m   \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39;49m(type_value, DType):\n\u001b[1;32m    820\u001b[0m     \u001b[39mif\u001b[39;00m type_value\u001b[39m.\u001b[39m_handle_data \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:  \u001b[39m# pylint:disable=protected-access\u001b[39;00m\n\u001b[1;32m    821\u001b[0m       \u001b[39mreturn\u001b[39;00m _INTERN_TABLE[type_value\u001b[39m.\u001b[39mas_datatype_enum]\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/abc.py:98\u001b[0m, in \u001b[0;36mABCMeta.__instancecheck__\u001b[0;34m(cls, instance)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__instancecheck__\u001b[39m(\u001b[39mcls\u001b[39m, instance):\n\u001b[1;32m     97\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Override for isinstance(instance, cls).\"\"\"\u001b[39;00m\n\u001b[0;32m---> 98\u001b[0m     \u001b[39mreturn\u001b[39;00m _abc_instancecheck(\u001b[39mcls\u001b[39m, instance)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "epochs = 20\n",
    "instance_PPO_Agent.explore(epochs)\n",
    "plt.plot(instance_PPO_Agent.history_training[2])\n",
    "plt.show()\n",
    "plt.axhline(0,color='r')\n",
    "plt.plot(instance_PPO_Agent.history_training[0][0:],label='actor_loss')\n",
    "plt.plot(instance_PPO_Agent.history_training[1][0:],label='critic_loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "total_episode += epochs\n",
    "instance_PPO_Agent.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (500, 500) to (512, 512) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n",
      "[swscaler @ 0x589ba80] Warning: data is not aligned! This can lead to a speed loss\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Video saved to test_video.mp4 with reward 35.095542581428276.\n"
     ]
    }
   ],
   "source": [
    "instance_PPO_Agent.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_PPO_Agent.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch: 1. Mean Return: 36.27247176550794. Mean Length: 60.23529411764706\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 10\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m instance_PPO_Agent\u001b[39m.\u001b[39;49mexplore(\u001b[39m720\u001b[39;49m)\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m plt\u001b[39m.\u001b[39mplot(instance_PPO_Agent\u001b[39m.\u001b[39mhistory_training[\u001b[39m2\u001b[39m])\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m plt\u001b[39m.\u001b[39mshow()\n",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 10\u001b[0m in \u001b[0;36mPPO_Agent.explore\u001b[0;34m(self, epochs)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=134'>135</a>\u001b[0m \u001b[39mfor\u001b[39;00m t \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msteps_per_epoch):\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=135'>136</a>\u001b[0m     state \u001b[39m=\u001b[39m state\u001b[39m.\u001b[39mreshape(\u001b[39m1\u001b[39m,\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m--> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=136'>137</a>\u001b[0m     action,action_distribution \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_action_and_distribution(state)\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=137'>138</a>\u001b[0m     value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minstance_PPO_Value_Model(state)\n\u001b[1;32m    <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=138'>139</a>\u001b[0m     log_probability \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_log_probability(action_distribution,action)\n",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 10\u001b[0m in \u001b[0;36mPPO_Agent.get_action_and_distribution\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=93'>94</a>\u001b[0m policy_mean,policy_standard_deviation \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minstance_PPO_Policy_Model(state)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=94'>95</a>\u001b[0m distribution_action \u001b[39m=\u001b[39m tfp\u001b[39m.\u001b[39mdistributions\u001b[39m.\u001b[39mNormal(policy_mean[\u001b[39m0\u001b[39m],policy_standard_deviation[\u001b[39m0\u001b[39m])\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=95'>96</a>\u001b[0m action \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39msqueeze(distribution_action\u001b[39m.\u001b[39;49msample(\u001b[39m1\u001b[39;49m),axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#W4sZmlsZQ%3D%3D?line=96'>97</a>\u001b[0m \u001b[39mreturn\u001b[39;00m [action\u001b[39m.\u001b[39mnumpy(),distribution_action]\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/distributions/distribution.py:1205\u001b[0m, in \u001b[0;36mDistribution.sample\u001b[0;34m(self, sample_shape, seed, name, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Generate samples of the specified shape.\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \n\u001b[1;32m   1192\u001b[0m \u001b[39mNote that a call to `sample()` without arguments will generate a single\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1202\u001b[0m \u001b[39m  samples: a `Tensor` with prepended dimensions `sample_shape`.\u001b[39;00m\n\u001b[1;32m   1203\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   1204\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_name_and_control_scope(name):\n\u001b[0;32m-> 1205\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_sample_n(sample_shape, seed, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/distributions/distribution.py:1182\u001b[0m, in \u001b[0;36mDistribution._call_sample_n\u001b[0;34m(self, sample_shape, seed, **kwargs)\u001b[0m\n\u001b[1;32m   1178\u001b[0m sample_shape \u001b[39m=\u001b[39m ps\u001b[39m.\u001b[39mconvert_to_shape_tensor(\n\u001b[1;32m   1179\u001b[0m     ps\u001b[39m.\u001b[39mcast(sample_shape, tf\u001b[39m.\u001b[39mint32), name\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39msample_shape\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m   1180\u001b[0m sample_shape, n \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_expand_sample_shape_to_vector(\n\u001b[1;32m   1181\u001b[0m     sample_shape, \u001b[39m'\u001b[39m\u001b[39msample_shape\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m-> 1182\u001b[0m samples \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sample_n(\n\u001b[1;32m   1183\u001b[0m     n, seed\u001b[39m=\u001b[39;49mseed() \u001b[39mif\u001b[39;49;00m callable(seed) \u001b[39melse\u001b[39;49;00m seed, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1184\u001b[0m samples \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39mnest\u001b[39m.\u001b[39mmap_structure(\n\u001b[1;32m   1185\u001b[0m     \u001b[39mlambda\u001b[39;00m x: tf\u001b[39m.\u001b[39mreshape(x, ps\u001b[39m.\u001b[39mconcat([sample_shape, ps\u001b[39m.\u001b[39mshape(x)[\u001b[39m1\u001b[39m:]], \u001b[39m0\u001b[39m)),\n\u001b[1;32m   1186\u001b[0m     samples)\n\u001b[1;32m   1187\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_set_sample_static_shape(samples, sample_shape, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow_probability/python/distributions/normal.py:181\u001b[0m, in \u001b[0;36mNormal._sample_n\u001b[0;34m(self, n, seed)\u001b[0m\n\u001b[1;32m    177\u001b[0m shape \u001b[39m=\u001b[39m ps\u001b[39m.\u001b[39mconcat([[n], \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_batch_shape_tensor(loc\u001b[39m=\u001b[39mloc, scale\u001b[39m=\u001b[39mscale)],\n\u001b[1;32m    178\u001b[0m                   axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[1;32m    179\u001b[0m sampled \u001b[39m=\u001b[39m samplers\u001b[39m.\u001b[39mnormal(\n\u001b[1;32m    180\u001b[0m     shape\u001b[39m=\u001b[39mshape, mean\u001b[39m=\u001b[39m\u001b[39m0.\u001b[39m, stddev\u001b[39m=\u001b[39m\u001b[39m1.\u001b[39m, dtype\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdtype, seed\u001b[39m=\u001b[39mseed)\n\u001b[0;32m--> 181\u001b[0m \u001b[39mreturn\u001b[39;00m sampled \u001b[39m*\u001b[39;49m scale \u001b[39m+\u001b[39m loc\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[39mreturn\u001b[39;00m fn(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    151\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/ops/math_ops.py:1466\u001b[0m, in \u001b[0;36m_OverrideBinaryOperatorHelper.<locals>.binary_op_wrapper\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m   1461\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1462\u001b[0m   \u001b[39m# force_same_dtype=False to preserve existing TF behavior\u001b[39;00m\n\u001b[1;32m   1463\u001b[0m   \u001b[39m# TODO(b/178860388): Figure out why binary_op_wrapper and\u001b[39;00m\n\u001b[1;32m   1464\u001b[0m   \u001b[39m#   r_binary_op_wrapper use different force_same_dtype values.\u001b[39;00m\n\u001b[1;32m   1465\u001b[0m   x, y \u001b[39m=\u001b[39m maybe_promote_tensors(x, y)\n\u001b[0;32m-> 1466\u001b[0m   \u001b[39mreturn\u001b[39;00m func(x, y, name\u001b[39m=\u001b[39;49mname)\n\u001b[1;32m   1467\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mTypeError\u001b[39;00m, \u001b[39mValueError\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m   1468\u001b[0m   \u001b[39m# Even if dispatching the op failed, the RHS may be a tensor aware\u001b[39;00m\n\u001b[1;32m   1469\u001b[0m   \u001b[39m# object that can implement the operator with knowledge of itself\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1472\u001b[0m   \u001b[39m# original error from the LHS, because it may be more\u001b[39;00m\n\u001b[1;32m   1473\u001b[0m   \u001b[39m# informative.\u001b[39;00m\n\u001b[1;32m   1474\u001b[0m   \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(\u001b[39mtype\u001b[39m(y), \u001b[39m\"\u001b[39m\u001b[39m__r\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m__\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m op_name):\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/ops/math_ops.py:1847\u001b[0m, in \u001b[0;36m_mul_dispatch\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m   1845\u001b[0m   \u001b[39mreturn\u001b[39;00m sparse_tensor\u001b[39m.\u001b[39mSparseTensor(y\u001b[39m.\u001b[39mindices, new_vals, y\u001b[39m.\u001b[39mdense_shape)\n\u001b[1;32m   1846\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1847\u001b[0m   \u001b[39mreturn\u001b[39;00m multiply(x, y, name\u001b[39m=\u001b[39;49mname)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[39mreturn\u001b[39;00m fn(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    151\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:1176\u001b[0m, in \u001b[0;36madd_dispatch_support.<locals>.decorator.<locals>.op_dispatch_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m   1174\u001b[0m \u001b[39m# Fallback dispatch system (dispatch v1):\u001b[39;00m\n\u001b[1;32m   1175\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1176\u001b[0m   \u001b[39mreturn\u001b[39;00m dispatch_target(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1177\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mTypeError\u001b[39;00m, \u001b[39mValueError\u001b[39;00m):\n\u001b[1;32m   1178\u001b[0m   \u001b[39m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[39;00m\n\u001b[1;32m   1179\u001b[0m   \u001b[39m# TypeError, when given unexpected types.  So we need to catch both.\u001b[39;00m\n\u001b[1;32m   1180\u001b[0m   result \u001b[39m=\u001b[39m dispatch(op_dispatch_handler, args, kwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/ops/math_ops.py:530\u001b[0m, in \u001b[0;36mmultiply\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m    481\u001b[0m \u001b[39m@tf_export\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mmath.multiply\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mmultiply\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    482\u001b[0m \u001b[39m@dispatch\u001b[39m\u001b[39m.\u001b[39mregister_binary_elementwise_api\n\u001b[1;32m    483\u001b[0m \u001b[39m@dispatch\u001b[39m\u001b[39m.\u001b[39madd_dispatch_support\n\u001b[1;32m    484\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmultiply\u001b[39m(x, y, name\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[1;32m    485\u001b[0m \u001b[39m  \u001b[39m\u001b[39m\"\"\"Returns an element-wise x * y.\u001b[39;00m\n\u001b[1;32m    486\u001b[0m \n\u001b[1;32m    487\u001b[0m \u001b[39m  For example:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    527\u001b[0m \u001b[39m   * InvalidArgumentError: When `x` and `y` have incompatible shapes or types.\u001b[39;00m\n\u001b[1;32m    528\u001b[0m \u001b[39m  \"\"\"\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m   \u001b[39mreturn\u001b[39;00m gen_math_ops\u001b[39m.\u001b[39;49mmul(x, y, name)\n",
      "File \u001b[0;32m~/anaconda3/envs/CX/lib/python3.8/site-packages/tensorflow/python/ops/gen_math_ops.py:6577\u001b[0m, in \u001b[0;36mmul\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m   6575\u001b[0m \u001b[39mif\u001b[39;00m tld\u001b[39m.\u001b[39mis_eager:\n\u001b[1;32m   6576\u001b[0m   \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 6577\u001b[0m     _result \u001b[39m=\u001b[39m pywrap_tfe\u001b[39m.\u001b[39;49mTFE_Py_FastPathExecute(\n\u001b[1;32m   6578\u001b[0m       _ctx, \u001b[39m\"\u001b[39;49m\u001b[39mMul\u001b[39;49m\u001b[39m\"\u001b[39;49m, name, x, y)\n\u001b[1;32m   6579\u001b[0m     \u001b[39mreturn\u001b[39;00m _result\n\u001b[1;32m   6580\u001b[0m   \u001b[39mexcept\u001b[39;00m _core\u001b[39m.\u001b[39m_NotOkStatusException \u001b[39mas\u001b[39;00m e:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "instance_PPO_Agent.explore(720)\n",
    "plt.plot(instance_PPO_Agent.history_training[2])\n",
    "plt.show()\n",
    "plt.axhline(0,color='r')\n",
    "plt.plot(instance_PPO_Agent.history_training[0][0:],label='actor_loss')\n",
    "plt.plot(instance_PPO_Agent.history_training[1][0:],label='critic_loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "#instance_PPO_Agent.test()\n",
    "video = []\n",
    "state = instance_PPO_Agent.instance_environment.reset()\n",
    "episodic_reward = 0\n",
    "while True:\n",
    "    state = state.reshape(1,-1)\n",
    "    action,action_distribution = instance_PPO_Agent.get_action_and_distribution(state)\n",
    "    state_next,reward,done,info = instance_PPO_Agent.instance_environment.step(action)\n",
    "    reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']\n",
    "    state = state_next\n",
    "    video.append(instance_PPO_Agent.instance_environment.render(mode ='rgb_array'))\n",
    "    if done:\n",
    "        break\n",
    "writer = imageio.get_writer('test_video.mp4',fps=30) \n",
    "for image in video:\n",
    "    writer.append_data(np.uint8(image))\n",
    "writer.close()\n",
    "print('Video saved to test_video.mp4.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (500, 500) to (512, 512) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n",
      "[swscaler @ 0x6cefa80] Warning: data is not aligned! This can lead to a speed loss\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Video saved to test_video.mp4.\n"
     ]
    }
   ],
   "source": [
    "video = []\n",
    "state = instance_PPO_Agent.instance_environment.reset()\n",
    "episodic_reward = 0\n",
    "while True:\n",
    "    state = state.reshape(1,-1)\n",
    "    action,action_distribution = instance_PPO_Agent.get_action_and_distribution(state)\n",
    "    state_next,reward,done,info = instance_PPO_Agent.instance_environment.step(action)\n",
    "    reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']\n",
    "    state = state_next\n",
    "    video.append(instance_PPO_Agent.instance_environment.render(mode ='rgb_array'))\n",
    "    if done:\n",
    "        break\n",
    "writer = imageio.get_writer('test_video.mp4',fps=30) \n",
    "for image in video:\n",
    "    writer.append_data(np.uint8(image))\n",
    "writer.close()\n",
    "print('Video saved to test_video.mp4.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Epoch: 1. Mean Return: -7.495318304387197. Mean Length: 64.0\n",
      " Epoch: 2. Mean Return: -5.257396168155204. Mean Length: 65.01587301587301\n",
      " Epoch: 3. Mean Return: -3.7793660920804037. Mean Length: 60.23529411764706\n",
      " Epoch: 4. Mean Return: -5.705623979676858. Mean Length: 59.36231884057971\n",
      " Epoch: 5. Mean Return: -2.273180657156479. Mean Length: 57.690140845070424\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD5CAYAAADCxEVRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAoRElEQVR4nO3deXzU1b3/8dcnOwEStoSEhLAIhD0BIlo3akVFZFFsrVq17W2l19Zea2trW3u7/GqttYttr7aKrb311q0qmgRFAatV6oIBMyHsOySTlUBWss7n90eCRQxhyXznOzP5PB+PeZDJTM75zCPknZMz53uOqCrGGGPCU4TbBRhjjHGOhbwxxoQxC3ljjAljFvLGGBPGLOSNMSaMWcgbY0wYi3KqYRH5JbAQaAV2AV9U1cM9fc2wYcN09OjRTpVkjDFhaf369dWqmtTdY+LUOnkRuQz4h6q2i8gvAFT1rp6+JicnRwsKChypxxhjwpWIrFfVnO4ec2y6RlVXqWp71913gXSn+jLGGNO9QM3J/wewsrsHRGSpiBSISEFVVVWAyjHGmL6hV3PyIrIGSOnmobtVNbfrOXcD7cAT3bWhqsuAZdA5XdObeowxxnxUr0JeVef29LiIfB5YAFyitkmOMcYEnJOra+YBdwFzVLXJqX6MMcacmJNz8g8CA4HVIlIoIg872JcxxphuODaSV9VxTrVtjDHm1NgVr8YY47LH1u5h1aZyR9q2kDfGGBc1t3Xw61XbWLOlwpH2LeSNMcZFr22ppLG1g8XZaY60byFvjDEuyi0sJWlgLOeOHepI+xbyxhjjktojbbyxrYqF00cQGSGO9GEhb4wxLnm1uJzWDh+Lskc41oeFvDHGuCTXU8qoofFkpSc61oeFvDHGuKCyrpl3dh1kcdYIRJyZqgELeWOMccWKojJ8iqNTNWAhb4wxrsj1eJmcmsC45IGO9mMhb4wxAbbvYCOeA4dZ7PAoHizkjTEm4PIKvQAsyLKQN8aYsKKqvFhYyuzRQ0gb1M/x/izkjTEmgDaX1bGrqtHxN1yPspA3xpgAyvN4iYoQ5k9LDUh/FvLGGBMgPp+SX+jlwvHDGNI/JiB9WsgbY0yAFOw7hLe22bEdJ7vjWMiLyE9FpKjr6L9VIhKYCShjjAlSeZ5S4qIjuHTy8ID16eRI/peqOl1Vs4EVwA8d7MsYY4JaW4ePl4rKmDtpOP1jHTt59WMcC3lVrTvmbn9AnerLGGOC3dod1RxqagvoVA04eJA3gIj8DLgZqAUuPsFzlgJLATIyMpwsxxhjXJPn8ZLYL5o5E5IC2m+vRvIiskZEiru5LQZQ1btVdSTwBHBbd22o6jJVzVHVnKSkwL54Y4wJhCOtHby6qZwrpqYQExXY9S69Gsmr6txTfOqTwEvAj3rTnzHGhKI1Wypoau0I2AVQx3Jydc34Y+4uArY61ZcxxgSzPI+X4QmxnDPGmXNce+LknPx9IpIJ+IB9wH862JcxxgSl2qY23thWyc2fGO3YOa49cSzkVfUap9o2xphQsbK4jLYODci2wt2xK16NMcZBuYVexgzrz7Q0585x7YmFvDHGOKSirpl39xxkocPnuPbEQt4YYxyS7/GiCosCcDjIiVjIG2OMQ/I8XqamJTAueYBrNVjIG2OMA/ZUN1JUUsvirMBuY3A8C3ljjHFAXqEXEViQFZjDQU7EQt4YY/xMVcn1dJ7jmpro/DmuPbGQN8YYP9vkrWN3VWPAd5zsjoW8Mcb42dFzXK+YmuJ2KRbyxhjjTz6fku/xMmdCEoMDdI5rTyzkjTHGj97fW0NZbbMrO052x0LeGGP8KNfjpV90ZEDPce2JhbwxJqjsP9jEzsoGt8s4I63tPl7eWMalk4cTHxO4c1x7EhxVGGMMcKCmicUPrUWB1745h6EDYt0u6bS8taOKw01tru042R0byRtjgkJjSzu3PF5Au09paG7nZy9vcbuk03b0HNcLxwfPUaYW8sYY16kqdz7rYXtFPQ/eMJOvzBnL8g2lvL2z2u3STllTazurNlUwf1pqwM9x7YnjlYjInSKiIjLM6b6MMaHpf/6xk5XF5XzviknMmZDE1z81nowh8dz9YjHNbR1ul3dKVm+u4EhbR1BN1YDDIS8iI4FLgf1O9mOMCV2rNpXzm9XbWTIjjS9fOAaAuOhI7rlqKnuqG/njG7tcrvDU5Hu8pCTEMXv0ELdL+QinR/IPAN8B1OF+jDEhaHtFPXc8U0hWeiL3Lpn2kYM1LpqQxKKsEfzxjV1Bv9rmUGMrb2yrYmFWKhEunOPaE8dCXkQWAaWq6jnJ85aKSIGIFFRVVTlVjjEmyBxuauWWxwuIj43ikZtyiIuO/NhzfrBgEnHREdz9wkZUg3esuLK4nHafBsVeNcfrVciLyBoRKe7mthi4G/jhydpQ1WWqmqOqOUlJwfOOtDHGOe0dPm578gPKDjfz8I2zSEmM6/Z5yQPj+O4Vk3hvTw3PrS8JcJWnLs9Tytik/kwZkeB2KR/Tq5BX1bmqOvX4G7AbGAN4RGQvkA5sEBH3d+sxxrju5yu3snZnNfdcPZVZowb3+Nzrzh7JrFGDufflLdQ0tgaowlNXVnuE9/bUsMjFc1x74sh0japuVNVkVR2tqqOBEmCmqpY70Z8xJnQ8t76EP6/dwxfOG821OSNP+vyICOHeq6dR39zOvUG4dn6Fp8z1c1x7EjyLOY0xYe+D/Yf4/gsbOe+sodx95aRT/rrMlIEsvWgsz60v4Z1dBx2s8PTlebxMT09kbJJ757j2JCAh3zWiD52rGowxfldR18xX/m89wxNieeiGmURHnl78fLh2/oWNtLQHx9r5XVUNbCytDdpRPNhI3hgTAM1tHXzl/9bT0NLOozfnnNE+6/1iIvnpVVPZHURr5z88x3W6hbwxpo9SVe5+oZjCA4f5zbVZTEw58xUocyYksTBrBH94fRe7q9xdO6+q5Hm8nDtm6AlXBwUDC3ljjKMe+9dent9Qwu2XjGfe1NRet/ffCyYRGx3B3S8Uu7p2vri0jj3VjUFzOMiJWMgbYxyzdkc1P3tpM5dPGc7tl4z3S5uda+cn8s7ugyzfUOqXNs9EbmEp0ZHBcY5rTyzkjTGO2FvdyNee3MD45IH8+tpsv17uf/3ZGczMGMQ9L212Ze18h0/JL/IyZ0Iyg+LdP8e1Jxbyxhi/a+jaG14EHr05hwGx/j2fKCJCuHdJ59r5n7uwdn7dnhoq6lqCfqoGLOSNMX7m8yl3PFPI7upGHrphJhlD4x3pZ2JKAl++cCzPri/h3d2BXTuf5yklPiaSuZOSA9rvmbCQN8b41W9f28HqzRX84MpJnD/O2WMkbr9kPOmD+wV07XznOa7lXBZE57j2xELeGOM3KzeW8fvXdvCZWel84bzRjvd3dO38rqpGHvnnbsf7A3hzexW1R9qCcsfJ7ljIG2P8YktZHd/8u4cZGYO45+qpAdus6+LMZBZMT+XB13cGZO18rsfL4PhoLhgfGofdWcgbY3qtprFzb/iEflE8cuMsYqM+vje8k364YDKxURH84EVn1843trSzenM586elnva2DG4JjSqNMUGrrcPHV59YT2V9C4/clENyQuCv/kxOiOOueRN5e9dBXix0bu38mi0VNLf5QmaqBizkjTG9dM+Kzby7u4b7lkwje+Qg1+q4YXYGMzIG8dMVWzjk0Nr53EIvqYlx5JxkD/xgYiFvjDljT6/bz1/f2cctF45hycx0V2s5uu987ZE27lu51e/t1zS28ub2KhZljQi6c1x7YiFvjDkjBXtr+O/cYi4cP4y75k10uxwAJqUm8OULx/BMwQHe8/Pa+Zc3ltHu05C4AOpYFvLGmNPmPXyE//zbBtIG9ePB62cSFURvQn64dv7FYlrbfX5rN8/j5ayk/kxODb5zXHvi2HdGRH4sIqUiUth1m+9UX8aYwDm6N3xzWweP3pxDYny02yV9RHxMFD+9aio7KxtY9qZ/9p33Hj7Cuj01LM5OC8pzXHvi9K/fB1Q1u+v2ssN9GWMcpqrc9XwRxd5afvvZbMYPH+h2Sd26ODOZK6en8vt/7GRvdWOv28v3eIHgPce1J8HzN5YxJugte3M3uYVevnXpBOZOHu52OT360YLJxEb6Z+18nsdLVnoio4f191N1geN0yN8mIkUi8piIdLvmSESWikiBiBRUVVU5XI4x5ky9sa2S+17ZypXTUvnaxePcLuekkhPi+M68TNburCa30HvG7eysrGeTt45FIbQ2/li9CnkRWSMixd3cFgN/BM4CsoEy4NfdtaGqy1Q1R1VzkpKSelOOMcYhu6sa+PpTHzAxJYFffmZ6yMxL33DOKLJHDuKnKzZzuOnM1s4fPcd14fTen2rlhl6FvKrOVdWp3dxyVbVCVTtU1Qc8Csz2T8nGmECqa27jy48XEB0ZwbKbZoXEzotHRXatnT98pI1fvHL6a+ePnuN63llDXbmS1x+cXF1z7K+9q4Fip/oyxjijw6d84+lC9h9s4g+fm8nIIc7sDe+kySMS+PIFY3hq3QHe31tzWl9bVFLL3oNNIfmG61FOzsnfLyIbRaQIuBi4w8G+jDEO+PWqbfxjayU/WjSFc8cOdbucM3b73PGkDerH95dvPK2187mFXmIiI5g3JTSnasDBkFfVm1R1mqpOV9VFqlrmVF/GGP/L83j5wxu7uH52Bjeek+F2Ob0SHxPFPVdNZUdlA4++dWr7znf4lBVFXj6ZmRR01wKcDltCaYz5mP0Hm/jOcx7OHj2YnyyaEjJvtPbk4onJzJ+Wwu9f28G+gydfO//e7oNU1ofGOa49sZA3xnzMr1ZtA+B/rp9JTFT4xMSPFk4h+hTXzucWeukfE8klE4P7eoCTCZ/vnjHGL4pLa8nzePnSBWNISQzNFSUnMrxr7fxbO6rJ85x47XxLewcvF5dx+ZQU+sUE9gAUf7OQN8Z8xC9e2cqg+Gi+Mucst0txxOfOGUVW19r52qa2bp/zz21V1De3szDEp2rAQt4Yc4x/7azmrR3V3HbxOBLiQvfNxp50rp2fyqGmNu47wdr5XI+XIf1juGBcaJzj2hMLeWMM0Hnhzy9e2UraoH7ceO4ot8tx1JQRiXzpgjE8tW4/6/d9dO18Q0s7azZXcGUInePak9B/BcYYv3h5YzlFJbV889IJxEWH9jz0qfjGh2vni2nr+Pfa+dWby2lp94X8qpqjLOSNMbR1+Pjlq1vJHD6Qq2aE5kZcpys+Jor/t3gK2yrqP7J2PrfQS9qgfszKCJ1zXHtiIW+M4en3D7D3YBN3XZFJZAidX9pbl0wazhVTU/jdmh3sP9jEwYYW3tpRzcIQO8e1JxbyxvRxjS3t/G7NDmaPHsLFmclulxNwH66dzy3m5Y1ldPiUxWEyVQMW8sb0eY+t3UN1Qwt3XTExLK5sPV0piXF8+/JM3txexa9Xb2d88gAmpgTniVdnwkLemD6sprGVR97czWWThzNrVHjMQZ+JG88dRVZ6Ioeb2licPSKsftlZyBvThz34j500tbbznXmZbpfiqsgI4b5rpjNr1GCumZXudjl+FTq7/xtj/OpATRN/e3cf1+aMZFxy+ExPnKlJqQk8f+t5bpfhdzaSN6aPemD1dkTgG3MnuF2KcZCFvDF90JayOl4oLOWL54ffJmTmoxwNeRH5uohsE5FNInK/k30ZY07d/a9sZWBsFLeG6SZk5t8cm5MXkYuBxcB0VW0Rkb63ANeYIPTu7oO8vq2K710xMaRPPDKnxsmR/K3AfaraAqCqlQ72ZYw5BarKfSu3kpoYx+fPG+12OSYAnFxdMwG4UER+BjQDd6rq+w72Z4xfNLd1UFHXTHltM+V1zZTVNnOoqZXPzBrJuOQBbpfXK69uKqfwwGHuv2Z6n9iEzPQy5EVkDZDSzUN3d7U9GDgXOBv4u4iM1ePO3BKRpcBSgIyM0D4s2AS/hpZ2ymuPUF7bQlntEcprmyk7GuhdoV7T2PqxrxOB/EIvubddQNLAWBcq7732Dh/3v7qN8ckDWDKzb2xCZnoZ8qo690SPicitwPKuUF8nIj5gGFB1XBvLgGUAOTk5PR+6aMwJqCqHm9ooq22mvK4zxMtrj3Td/3eI17e0f+xrh/SPISUhjtTEOGZkDCIlIY6UxDhSE/uRktj58d7qRj798Nvc+rf1PHnLuSF57umz60vYXdXIsptmERUG+6SbU+PkdM2LwKeAN0RkAhADVDvYnwlzzW0dvLm9Cu/hI5TXfTzEW9p9H3l+hEDywM6QPitpAOePG0ZqV2h3hno/khNiT2naYmpaIr/8dBZff+oDfpRXzL1XTwupS9+PtHbwwOrtzBo1mEsnh/bB1Ob0OBnyjwGPiUgx0Ap8/vipGmNOx0Ov7+R//rETgJjICIYnxpKa0I+s9EFcPiXuw9H40dF30oBYv45YF2aNYGt5HQ+9votJqQnc/InRfmvbaX95ew+V9S089LmZIfXLyfSeYyGvqq3AjU61b/oWVSW30Mu5Y4fw0A0zGdI/xpWw+talmWwrr+cn+ZsZlzSA80LgDNDDTa388Y1dzJ2UzNmjh7hdjgkwm5gzIcFTUsv+miaWzExn6IBY10ajERHCA5/NZuyw/nz1yQ3sP9jkSh2n4w9v7KKhpZ1vXz7R7VKMCyzkTUjI93iJiYzg8indLeYKrIFx0Tx6cw6qcMvjBTR082ZusCg9fIT/fXsv18xMJzOM9kg3p85C3gS9Dp+yosjLnMwkEvsFxxWao4f156EbZrKzqoFvPlOIzxecbzc9sHo7AHdcapuQ9VUW8ibovb+3hoq6FhZmBdeRbBeMH8bd8yexanMFv31th9vlfMy28nqWbyjh858YRdqgfm6XY1xi+8mboJfv8dIvOpK5k4Jv+6Mvnj+aLWV1/P61HWQOH8iV01PdLulDv3x1K/1jo/jqJ8e5XYpxkY3kTVBr6/Dx8sYyLpmUTHxM8I1JRIR7rp7KzIxB3Pmsh03eWrdLAjr/+lmzpZL/nHMWg/vHuF2OcZGFvAlq/9pZzaGmNhYF2VTNsWKjInn4plkk9otm6ePrOdjQ4mo9RzchSx4Yy3+cP8bVWoz7LORNUMv3lDEwLoo5mUlul9Kj5IFxLLt5FtUNLdz6xAZaj7v6NpDWbKlk/b5DfGPuBPrF2CZkfZ2FvAlazW0drNpUzuVTUoiNCv6wmp4+iPs/PZ11e2r4Sf4mV2po7/Bx/ytbGTusP9fmhNeB1ObMBN8kpzFd3thWRX1Le1BP1RxvcXYaW8rqefifu5iYmsBN544KaP/LN5Syo7KBh2+caZuQGcBG8iaI5Rd5GdI/hvPOGup2Kafl25dn8qmJyfwkbxPv7j4YsH6b2zp4YM12skcOCoqLxkxwsJA3QamxpZ3XtlQwf1pKyI1IIyOE316Xzaih8Xz1iQ0cqAnM1gd/fXsvZbXNfPeKibYJmflQaP30mD5jzZYKmtt8LMoKzcMtErq2Pmjr8HHL4wU0Orz1QW1TG394YxefzEzi3LGh9ZePcZaFvAlK+R4vKQlx5Iwa7HYpZ2xs0gAevGEm2yvqufNZj6NbH/zxn7uoa27jO7YJmTmOhbwJOrVNbfxzexULpqcSERHa0w5zJiTx/fmTWFlc/uFe+P5WVnuEv/xrD1dlpzF5RIIjfZjQZatrTNB5ZVMZbR0adHvVnKkvXTCGzWV1PLBmO5kpA5g31b9bH/xuzQ5U4Zu2CZnpho3kTdDJ95Qxamg809MT3S7FL0SEe6+eRvbIQXzz7x62ltf5re2dlfX8veAAN547ipFD4v3WrgkfjoW8iDwjIoVdt70iUuhUXyZ8VNW38PauahZOHxFWK0TioiN55KZZDIiN4st/LaCmsdUv7f7y1W3Ex0Rx26dsEzLTPcdCXlU/q6rZqpoNPA8sd6ovEz5WFpfhU8JmquZYwxPiWHZzDpX1LXz1ifW0dfRu64P1+w7x6qYKvnLRWIbYJmTmBByfrpHO4di1wFNO92VCX16hl8zhA8P2FKPskYP4xTXTeHd3DT9dsfmM21FVfrFyK8MGxPKlC20TMnNigZiTvxCoUNVuT1UQkaUiUiAiBVVVVQEoxwSr0sNHKNh3iIVZwbMnuxOunpHO0ovG8vg7+3jyvf1n1Mbr2ypZt7eG2+eOD8otmE3w6FXIi8gaESnu5rb4mKddTw+jeFVdpqo5qpqTlBTcOw0aZ71U5AVgwfTwm6o53l3zJjJnQhI/zC1m3Z6a0/raDp/yi5XbGD00nuvOHulQhSZc9CrkVXWuqk7t5pYLICJRwBLgGX8Ua8JbnsdLVnoio4f1d7sUx0VGCL+/fgYZQ+K59W/rKTl06lsfvPhBKdsq6rnz8kyiQ2zLBxN4Tv8PmQtsVdUSh/sxIW53VQPFpXVh+YbriST2i+bRz+fQ2uFj6ePraWo9+dYHzW0d/Gb1dqanJzLfz+vtTXhyOuSvw95wNadgRVEZQFCdkRoIZyUN4PfXz2BLeR3ffrYI1Z63Pvjbu/soPXyEu+ZNDPmrgU1gOBryqvoFVX3YyT5M6FNV8jxeZo8eQmpiP7fLCbiLM5P57ryJvLSxjIdeP/HWB3XNbTz4+k4uHD+M88cNC2CFJpTZhJ5x3dbyenZWNrAwu+9M1Rxv6UVjuXpGGr9atZ1Vm8q7fc6yf+7mcFMbd82zTcjMqbOQN67L93iJjBCumNp3D7oQEX6+ZBrT0xO545lCtpXXf+Txyrpm/rR2N4uyRjA1LTy2ezCBYSFvXKWq5Bd5Oe+soQwbEOt2Oa6Ki45k2U05xMdGccvjBRw6ZuuD3762g/YO5VuX2SZk5vRYyBtXFR44zIGaIyF1jquTUhLjeOSmWZTXNnPbUxto7/Cxu6qBZ94/wOfOyWDU0PBfXmr8yy6VM67K95QRExnBZXYm6YdmZgzm3iXTuPNZD/e8tIXK+mbioiL4+iXj3S7NhCALeeOaDp+yosjLnMwkEvtFu11OUPn0rHS2lNXx57V7ALj9kvF9fjrLnBkLeeOadXtqqKxvsamaE/jeFRPZWdnA9op6brlorNvlmBBlIW9ck1/kpV90JJdMSna7lKAUFRnBX75wNkfaOugfaz+q5szYG6/GFW0dPlZuLGPu5OG2i2IPIiLEAt70ioW8ccXandUcampjYR/bxsCYQLOQN67I93gZGBfFnEzbXtoYJ1nIm4Brbutg1aYK5k1JITYq0u1yjAlrFvIm4N7YVkVDS3uf2lbYGLdYyJuAy/d4Gdo/hvPOGup2KcaEPQt5E1ANLe28trWC+dNSibJTjYxxnP2UmYB6bUsFzW0+m6oxJkAcC3kRyRaRd0WkUEQKRGS2U32Z0JFX6CU1MY6cUYPdLsWYPsHJkfz9wE9UNRv4Ydd904cdbmrlzR1VLJieakfXGRMgToa8AgldHycCXgf7MiHg1U3ltHWoTdUYE0BOXi/9DeBVEfkVnb9MzuvuSSKyFFgKkJGR4WA5xm15Hi+jhsYzzU42MiZgejWSF5E1IlLczW0xcCtwh6qOBO4A/txdG6q6TFVzVDUnKcmufgxXlfXNvLPrIIuyRiBiUzXGBEqvRvKqOvdEj4nI48DtXXefBf7Um75MaFu5sRyfYlM1xgSYk3PyXmBO18efAnY42JcJcvkeL5nDBzJh+EC3SzGmT3FyTv4W4HciEgU00zXvbvqe0sNHKNh3iG9fnul2Kcb0OY6FvKquBWY51b4JHSs8nQurFti2wsYEnF3xahyXX+QlKz2RUUP7u12KMX2Ohbxx1O6qBopL6+wNV2NcYiFvHJXvKUMEFky3kDfGDRbyxjGqSp6nlLNHDyElMc7tcozpkyzkjWO2lNWzq6rRpmqMcZGFvHFMfpGXyAhh/tQUt0sxps+ykDeOUFXyPV7OHzeMoQNi3S7HmD7LQt44ovDAYUoOHWGhrY03xlUW8sYReR4vMZERXG5TNca4ykLe+F2HT3mpqIxPZiaREBftdjnG9GkW8sbv1u2pobK+xVbVGBMELOSN3+V5vMTHRHLJpGS3SzGmz7OQN37V1uFjZXEZcycNJz7GyU1OjTGnwkLe+NXandUcbmqzqRpjgoSFvPGrfI+XhLgoLpowzO1SjDFYyBs/am7rYNWmCuZNTSE2KtLtcowxOBjyIpIlIu+IyEYRyReRBKf6MsHhjW2VNLS021SNMUHEyZH8n4Dvquo04AXg2w72ZYJAvqeMYQNi+MTYoW6XYozp4mTIZwJvdn28GrjGwb6Myxpa2lmzpYL501KJirRZQGOChZM/jcXAoq6PPwOM7O5JIrJURApEpKCqqsrBcoyT1myuoKXdZ1M1xgSZXoW8iKwRkeJubouB/wC+JiLrgYFAa3dtqOoyVc1R1ZykpKTelGNclO/xkpoYx6yMwW6XYow5Rq+uVlHVuSd5ymUAIjIBuLI3fZngdbiplTd3VPHF88cQESFul2OMOYaTq2uSu/6NAH4APOxUX8ZdrxSX09ahLLRzXI0JOk7OyV8vItuBrYAX+IuDfRkX5Rd5GT00nqlptkrWmGDj2OYiqvo74HdOtW+CQ2V9M+/sOsjXLh6HiE3VGBNsbK2b6ZWXi8rwKSyyVTXGBCULedMr+UVlTEwZyPjhA90uxRjTDQt5c8ZKDjWxft8hWxtvTBCzkDdnbEVRGYCtqjEmiFnImzOW7/GSNXIQGUPj3S7FGHMCFvLmjOyqamCTt46F01PdLsUY0wMLeXNG8j1eRGCBTdUYE9Qs5M1pU1XyPV5mjx5CSmKc2+UYY3pgIW9O25ayenZVNdqqGmNCgGNXvIa76oYWXt5YxgpPGdFRwrU5I7l8Sgpx0eF/7F1+kZfICOGKqSlul2KMOQkL+dPQ2NLOqs3l5BZ6eWtHNR0+JXP4QI60dXD704UMio/mmpnpXD97JOOSw+viIFVl/b5DPL+hlNzCUs4fN4yhA2LdLssYcxIW8ifR2u7jrR1VvFjoZfXmcprbfKQN6sfSi8ayOHsEE1MS8PmUt3cd5Kn39/P4O3v589o95IwazPWzM5g/LZV+MaE7ut9/sInlH5Twwgel7DvYRHxMJPOmpvCNSya4XZox5hSIqrpdw4dycnK0oKDA7TLw+ZT1+w/x4gelvLyxjENNbQyKj+bKaalcNSONWRmDT7hv+sGGFp7fUMLT6w6wu7qRgXFRLJmRxnWzM5iUGhq7NNY1t/FyURnLN5Sybm8NInDeWUNZMiOdeVNT6B9rYwNjgomIrFfVnG4fs5D/t63ldeQWeskr9FJ6+Ahx0RFcNjmFxdkjuHB8EjFRp/4+tary3p4anl63n5eLy2lt95E1chA3zB7Jgukjgi4o2zt8vLWzmuUbSlm1qZyWdh9jk/pzzcx0rpqRRtqgfm6XaIw5AQv5HpQePkJeoZfcwlK2ltcTGSFcOH4Yi7NHcNlk/4xaDze1snxDKU+t28+OygYGxEaxKHsE15+dwbT0RD+8ijO3payO5RtKeLHQS1V9C4Pio1mUNYIlM9PJSk+07YONCQEW8sc51NjKSxvLyCv0sm5vDQAzMwZx1Yw05k9LZZhDbyiqKhv2H+KpdQdYUeSluc3H1LQErjs7g8XZIxgYF+1Iv8erqm8ht7CU5zeUsqWsjuhI4eLMZJbMTOdTE5NP6y8WY4z7LOSBI60drN5SQe4HpfxzexXtPmVc8gCuyh7Boqy0gO+/UnukjbzCUp5cd4AtZXX0i45kYVYq183OYMbIQX4fQTe3dbBmSwXPry/hza6VQVnpiSyZmc7CrBEM6R/j1/6MMYHjWMiLyGeAHwOTgNmqWnDMY98DvgR0AP+lqq+erD1/h3x7h4+1O6vJLfTy6qZymlo7SEmIY1H2CBZnj2ByaoLr0xGqSlFJLU+t20+ex0tTawcTUwZy3dkjuXpGOonxZz66V1UK9h1i+YYSVhSVUd/cTmpiHFfPSGPJzLSwW+ZpTF/lZMhPAnzAI8CdR0NeRCYDTwGzgRHAGmCCqnb01J4/Ql5V+eDAYXI/KGVFURkHG1tJiIti/rRUFmencc6YISdcGeO2hpZ28gq9PP3+fopKaomNiuDKaZ2j+7NHDz7lX0hHlz0u31DK/pp/L3u8ZmY6544dSmSQvn5jzJnpKeR79a6iqm7p6uD4hxYDT6tqC7BHRHbSGfjv9Ka/nuysrCe30EtuoZf9NU3ERkUwd9JwFmWP4JOZScRGBf9a9QGxUdxwTgY3nJNBcWktT7+/n9wPvCz/oJSzkvpz/ewMlsxM73Zqpbtlj58YO5TbLxlvyx6N6cP8MicvIm/w0ZH8g8C7qvq3rvt/Blaq6nPdfO1SYClARkbGrH379p12/2/vquaGR98jQuD8ccNYnJ3G5VOGB+yNTCc1tbazoqiMp9ftZ8P+w8RERnD51BSunz2Ss0cPYe3Oap5fX8LqzRW27NGYPqpXI3kRWQN0t0nJ3aqae6Iv6+Zz3f42UdVlwDLonK45WT3dyRk1hB8vnMz8aakkJ4TXrojxMVFcmzOSa3NGsq28nqfW7Wf5hhLyPV5ioiJobfcxKD6az5490pY9GmM+5qQhr6pzz6DdEmDkMffTAe8ZtHNKYqIi+ML5Y5xqPmhkpgzkx4um8N0rJrKyuIx1ew4xZ0KSLXs0xpyQUxO1ecCTIvIbOt94HQ+sc6ivPicuOpKrZ6Rz9Yx0t0sxxgS5Xg3/RORqESkBPgG8JCKvAqjqJuDvwGbgFeBrJ1tZY4wxxv96u7rmBeCFEzz2M+BnvWnfGGNM79hErjHGhDELeWOMCWMW8sYYE8Ys5I0xJoxZyBtjTBizkDfGmDAWVPvJi0gVcPqb1/zbMKDaT+WEgr72esFec19hr/n0jFLVpO4eCKqQ7y0RKTjRJj3hqK+9XrDX3FfYa/Yfm64xxpgwZiFvjDFhLNxCfpnbBQRYX3u9YK+5r7DX7CdhNSdvjDHmo8JtJG+MMeYYFvLGGBPGwiLkRWSeiGwTkZ0i8l2363GaiIwUkddFZIuIbBKR292uKVBEJFJEPhCRFW7XEggiMkhEnhORrV3f70+4XZOTROSOrv/TxSLylIiE13meXUTkMRGpFJHiYz43RERWi8iOrn8H+6OvkA95EYkEHgKuACYD14vIZHerclw78C1VnQScC3ytD7zmo24HtrhdRAD9DnhFVScCWYTxaxeRNOC/gBxVnQpEAte5W5Vj/heYd9znvgu8pqrjgde67vdayIc8MBvYqaq7VbUVeBpY7HJNjlLVMlXd0PVxPZ0/+GnuVuU8EUkHrgT+5HYtgSAiCcBFwJ8BVLVVVQ+7WpTzooB+IhIFxOPg2dBuUtU3gZrjPr0Y+GvXx38FrvJHX+EQ8mnAgWPul9AHAu8oERkNzADec7mUQPgt8B3A53IdgTIWqAL+0jVF9ScR6e92UU5R1VLgV8B+oAyoVdVV7lYVUMNVtQw6B3JAsj8aDYeQl24+1yfWhYrIAOB54BuqWud2PU4SkQVApaqud7uWAIoCZgJ/VNUZQCN++hM+GHXNQS8GxgAjgP4icqO7VYW+cAj5EmDkMffTCdM/8Y4lItF0BvwTqrrc7XoC4HxgkYjspXNK7lMi8jd3S3JcCVCiqkf/SnuOztAPV3OBPapapaptwHLgPJdrCqQKEUkF6Pq30h+NhkPIvw+MF5ExIhJD5xs1eS7X5CgRETrnabeo6m/cricQVPV7qpquqqPp/B7/Q1XDepSnquXAARHJ7PrUJcBmF0ty2n7gXBGJ7/o/fglh/EZzN/KAz3d9/Hkg1x+NRvmjETeparuI3Aa8Sue78Y+p6iaXy3La+cBNwEYRKez63PdV9WX3SjIO+TrwRNcAZjfwRZfrcYyqvicizwEb6FxB9gFhur2BiDwFfBIYJiIlwI+A+4C/i8iX6PyF9xm/9GXbGhhjTPgKh+kaY4wxJ2Ahb4wxYcxC3hhjwpiFvDHGhDELeWOMCWMW8sYYE8Ys5I0xJoz9f5VUzyj6uVXaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAobUlEQVR4nO3deXxU1d3H8c+PEBJ2gYAii6ClKmuwoFgUUbS4ICBVASko+Gi1olTrbl0rfewjLrVujcriCiiKaK0oIOCCCiguCKiASAQ1gqwCIcl5/jgzZAgTkpnMZDKT7/v1ymtm7ty598xJ8p0z5557rjnnEBGR1FIj0QUQEZHYU7iLiKQghbuISApSuIuIpCCFu4hICqqZ6AIAZGVluTZt2iS6GCIiSWXx4sU/OeeahnuuSoR7mzZtWLRoUaKLISKSVMxsTWnPqVtGRCQFKdxFRFKQwl1EJAVViT53EUkdu3fvJjc3l507dya6KCkjMzOTli1bkp6eXu7XKNxFJKZyc3OpX78+bdq0wcwSXZyk55xjw4YN5Obm0rZt23K/Tt0yIhJTO3fupEmTJgr2GDEzmjRpEvE3IYW7iMScgj22oqnP5A/3b96BvC8TXQoRkSol+fvcJ57hb2/bnNhyiIhUIcnfchcRqYC5c+fy3nvvxWRbEydOZPTo0THZVkUp3EWkWosm3AsKCuJUmthJ/m4ZEamybn9lKV+s2xLTbbY/uAG3ntmhzPUGDhzI2rVr2blzJ2PGjOHiiy/m9ddf58Ybb6SwsJCsrCyeeOIJHn30UdLS0nj66af517/+RevWrRk1ahR5eXk0bdqUCRMm0Lp1ay644AIaN27Mxx9/zFFHHcU999yz3/2vWbMm7Haef/55br/9dtLS0mjYsCHz589n6dKljBw5kvz8fIqKipg2bRrt2rWrUD0p3EUkJY0fP57GjRuzY8cOunfvzoABA7jooouYP38+bdu2ZePGjTRu3JhLLrmEevXqcfXVVwNw5plnMmLECM4//3zGjx/PFVdcwfTp0wH48ssvmTVrFmlpaWXuf/To0WG3c8cddzBz5kxatGjBpk2bAHj00UcZM2YMw4YNIz8/n8LCwgq/f4W7iMRNeVrY8fLAAw/w0ksvAbB27VpycnLo1avXnhOBGjduHPZ1CxYs4MUXXwRg+PDhXHvttXueO+ecc8oV7PvbTs+ePbngggs499xzGTRoEADHHnssY8eOJTc3l0GDBlW41Q7qcxeRFDR37lxmzZrFggUL+OSTT+jatStdunSJarx46Gvq1q0bdZmC23n00Ue58847Wbt2LdnZ2WzYsIHzzjuPGTNmULt2bfr27cucOXOi3k+Qwl1EUs7mzZtp1KgRderUYfny5bz//vvs2rWLefPmsXr1agA2btwIQP369dm6deue1/72t79l8uTJADzzzDMcd9xxUZWhtO2sXLmSY445hjvuuIOsrCzWrl3LqlWrOPTQQ7niiivo378/n376adTvPUjhLiIp59RTT6WgoIDOnTtz880306NHD5o2bUpOTg6DBg2iS5cuDB48GPB97C+99BLZ2dm8/fbbPPDAA0yYMIHOnTvz1FNP8c9//jOqMpS2nWuuuYZOnTrRsWNHevXqRZcuXZgyZQodO3YkOzub5cuXM2LEiArXgTnnKryRiurWrZuL+kpMtzUM3OokJpGqYNmyZRx55JGJLkbKCVevZrbYOdct3PpquYuIpCCNlhERidCECRP26a7p2bMnDz30UIJKtC+Fu4hIhEaOHMnIkSMTXYz9UreMiEgKUriLiKQghbuISApSuIuIpCCFu4hUSzNmzOCuu+4CYPr06XzxxRd7nrvllluYNWtWRNubO3cu/fr1i2kZK0KjZUSk2ikoKKB///70798f8OHer18/2rdvD8Add9yRyOLFhMJdROLnv9fD95/FdpsHdYLT7ipztSeffJJx48ZhZnTu3Jm0tLS95mPv1KkTixYt2jNp17x587jzzjuZNm0af/vb3+jXrx9nn302CxcuZMyYMWzfvp2MjAxmz55N/fr197vvjRs3MmrUKFatWkWdOnXIycmhc+fOzJs3jzFjxgB+IrH58+ezbds2Bg8ezJYtWygoKOCRRx7h+OOPr3A1KdxFJOUsXbqUsWPH8u6775KVlcXGjRu56qqr9pqPfeLEiYCf4Kt///57wjxUfn4+gwcPZsqUKXTv3p0tW7ZQu3btMvd/66230rVrV6ZPn86cOXMYMWIES5YsYdy4cTz00EP07NmTbdu2kZmZSU5ODn379uWmm26isLCQX375JSZ1oHAXkfgpRws7HubMmcPZZ59NVlYWUDx3eyTzsQOsWLGC5s2b0717dwAaNGhQrte98847TJs2DYCTTjqJDRs2sHnzZnr27MlVV13FsGHDGDRoEC1btqR79+6MGjWK3bt3M3DgQLKzsyN4p6XTAVURSTnOubBzt0c6H3tp2ynP60oyM66//noef/xxduzYQY8ePVi+fDm9evVi/vz5tGjRguHDh/Pkk09GvL9wFO4iknL69OnD1KlT2bBhA1A8d3tpSs7pHnTEEUewbt06Fi5cCMDWrVvLdXHsXr168cwzzwB+FE1WVhYNGjRg5cqVdOrUieuuu45u3bqxfPly1qxZQ7Nmzbjooou48MIL+eijjyJ9u2GpW0ZEUk6HDh246aabOOGEE0hLS6Nr1677XX/IkCFcdNFFPPDAA7zwwgt7lteqVYspU6Zw+eWXs2PHDmrXrs2sWbOoV6/efrd32223MXLkSDp37kydOnWYNGkSAPfffz9vvfUWaWlptG/fntNOO43Jkydz9913k56eTr169WLWctd87iISU5rPPT5iPp+7mbUys7fMbJmZLTWzMYHljc3sTTP7KnDbKOQ1N5jZ12a2wsz6VvA9iYhIhMrT514A/MU5dyTQA7jMzNoD1wOznXPtgNmBxwSeGwJ0AE4FHjaz8h+eFhGpwmbOnEl2dvZeP2eddVaii7WPMvvcnXPrgfWB+1vNbBnQAhgA9A6sNgmYC1wXWD7ZObcLWG1mXwNHAwtiXXgRqZqiHWWSDPr27UvfvpXbIRFN93lEo2XMrA3QFfgAODAQ/MEPgGaB1VoAa0NelhtYVnJbF5vZIjNblJeXF3HBRaRqyszMZMOGDVEFkuzLOceGDRvIzMyM6HXlHi1jZvWAacCfnXNb9vOpHO6JfX7LzrkcIAf8AdXylkNEqraWLVuSm5uLGm2xk5mZScuWLSN6TbnC3czS8cH+jHPuxcDiH8ysuXNuvZk1B34MLM8FWoW8vCWwLqJSiUjSSk9Pp23btokuRrVXntEyBjwBLHPO3Rvy1Azg/MD984GXQ5YPMbMMM2sLtAM+jF2RRUSkLOVpufcEhgOfmdmSwLIbgbuAqWZ2IfAtcA6Ac26pmU0FvsCPtLnMOVcY64KLiEjpyjNa5h3C96MD9CnlNWOBsRUol4iIVIDmlhERSUEKdxGRFKRwFxFJQckd7jpJQkQkLIW7iEgKSu5w3/fEVxERIdnDXS13EZGwkjvcRUQkrCQPd7XcRUTCSe5wV7eMiEhYyR3uarmLiISV3OGulruISFjJHe5quYuIhJXc4a6Wu4hIWMkd7mq5i4iEldzhrpa7iEhYyR3uarmLiISV3OGulruISFjJHe5quYuIhJXc4a6Wu4hIWMkd7mq5i4iEldzhrpa7iEhYyR3uIiISlsJdRCQFJXe4q1tGRCSs5A53HVAVEQkrucNdLXcRkbCSO9zVchcRCSu5w10tdxGRsJI73NVyFxEJK7nDXS13EZGwkjvc1XIXEQkrucNdLXcRkbDKDHczG29mP5rZ5yHLbjOz78xsSeDn9JDnbjCzr81shZn1jVfBPYW7iEg45Wm5TwRODbP8PudcduDnNQAzaw8MAToEXvOwmaXFqrD7UMtdRCSsMsPdOTcf2FjO7Q0AJjvndjnnVgNfA0dXoHxlULiLiIRTkT730Wb2aaDbplFgWQtgbcg6uYFl+zCzi81skZktysvLi64EarmLiIQVbbg/AhwGZAPrgXsCyy3MumET2DmX45zr5pzr1rRp0yiLISIi4UQV7s65H5xzhc65IuAxirtecoFWIau2BNZVrIj7LUn8Ni0iksSiCnczax7y8CwgOJJmBjDEzDLMrC3QDviwYkXcD3XLiIiEVbOsFczsOaA3kGVmucCtQG8zy8Y3nb8B/gjgnFtqZlOBL4AC4DLnXGFcSg6o5S4iEl6Z4e6cGxpm8RP7WX8sMLYihSo3tdxFRMJK7jNURUQkrOQO99CWe2FB4sohIlLFJHe4h/a5P9ozccUQEalikjvcQ1vuecsTVw4RkSomucNdo2VERMJK7nDXaBkRkbCSO9zVchcRCSu5w10tdxGRsJI73Eu23BX2IiJAsod7yTAv3J2YcoiIVDHJHe77tNzjOI2NiEgSSfJwL6FI4S4iAske7iW7ZdRyFxEBkj3cS3bLqOUuIgIke7jv03IvSkw5RESqmOQOd7XcRUTCSu5wV5+7iEhYyR3uarmLiISV3OFe8oRUtdxFRIBkD/d6zfZ+rJa7iAiQ7OF+QCs4dnTxY42WEREBkj3cAcyK76vlLiICpEK4h46YUZ+7iAiQCuEeSi13EREgFcI9tJ9dLXcRESAlwj2kW6ZIB1RFRCAlwl0tdxGRklIr3NXnLiICpEK4h1LLXUQESIVwT0svvq+Wu4gIkGrhrpa7iAiQCuFeI7TlrtEyIiKQCuGulruIyD7KDHczG29mP5rZ5yHLGpvZm2b2VeC2UchzN5jZ12a2wsz6xqvge9SoWXx/63rY+kPcdykiUtWVp+U+ETi1xLLrgdnOuXbA7MBjzKw9MAToEHjNw2aWFrPShpNWq/j+K2Pgnl/HdXciIsmgzHB3zs0HNpZYPACYFLg/CRgYsnyyc26Xc2418DVwdGyKWorQcBcRESD6PvcDnXPrAQK3watmtADWhqyXG1i2DzO72MwWmdmivLy8KIsBpNUsex0RkWom1gdULcyykhfD8wudy3HOdXPOdWvatGn0ewwdLSMiIkD04f6DmTUHCNz+GFieC7QKWa8lsC764pVDmsJdRKSkaMN9BnB+4P75wMshy4eYWYaZtQXaAR9WrIhlqN1432U6U1VEqrnyDIV8DlgAHG5muWZ2IXAXcIqZfQWcEniMc24pMBX4AngduMy5OA8+/3Vf6H3D3svyt8V1lyIiVV2ZRyOdc0NLeapPKeuPBcZWpFARMYPuF8Hc/y1elr8dMhtWWhFERKqa5D9DFfYdMbNLLXcRqd5SJNxLjHXP35qYcoiIVBGpGe5quYtINZca4V6jxAwH+dsTUw4RkSoiNcK9JI2WEZFqLjXDfZf63EWkekvNcFfLXUSqudQM9zdvgRf/mOhSiIgkTOqFe81Mf/vpZCgsSGxZREQSJPXCvWBn8f2NqxJXDhGRBEq9cA/18+pEl0BEJCFSPNzXJLoEIiIJkbrhnpYBP61IdClERBIida5Rd+6TkF4HPpkMPy6Duk1g4ePQsCUcd2WiSyciUqlSJ9zbD/C37U7xt2sWwOr58NbfocefoGZG4somIlLJUrdb5pBjYchzUJgPHz0J234s+zUiIikidcMdoO3x/va1qyGnNxQVJbQ4IiKVJbXDPaM+nHC9v7/lO8hbltjyiIhUktQOd4ATb4A/f+bvf/SkP9gqIpLiUj/cAQ5o7X8+eBQe7qEzV0Uk5VWPcAc46Rao3cjfX/hEYssiIhJn1SfcO58D130DHc6CBQ/CzJugID/RpRIRiYvqE+5BvW+Agzr7gH97XKJLIyISF9Uv3JseDpe8DZ2HwPxxsHhSokskIhJzqXOGaqRO/z/YngevXOHPXu0yJNElEhGJmerXcg/KbAhDn4O2veDly2DlnESXSEQkZqpvuINvsQ9+GpoeAVOG+/loRERSQPUOd/At+GEvQP2D4OlBOslJRFKCwh2gQXO44D9Qqy48NQh++irRJRIRqRCFe1D9g2DEy1C027fgt6xLdIlERKKmcA91YAc4byr88jNMOhO2fp/oEomIREXhXlKLo+APL/hgn3iGWvAikpQU7uG07gF/mAZbf/ABv/m7xJTjvX/BO/clZt8iktQqFO5m9o2ZfWZmS8xsUWBZYzN708y+Ctw2ik1RK1nrHjD8Jdj+E0w8HTatrfwyvPFXmHWbRvCISMRi0XI/0TmX7ZzrFnh8PTDbOdcOmB14nJxadYfh030f/MTT4ec1lbv/2o397cybwLnK3beIJLV4dMsMAIITtkwCBsZhH5Wn5W9gxHTYudl30VTmXPB1s/ztytmwbEbl7VdEkl5Fw90Bb5jZYjO7OLDsQOfceoDAbbMK7iPxWhwFI2bA5lw/2VhlKSqE9gPhwI7w+o2Qv73y9i0iSa2i4d7TOXcUcBpwmZn1Ku8LzexiM1tkZovy8vIqWIxKcHA2NGsPOzZV3j5doZ8i4fRxsKWSP1hEJKlVKNydc+sCtz8CLwFHAz+YWXOAwO2Ppbw2xznXzTnXrWnTphUpRuVJz4SCnZW3v6JCsDQ45FjoMtSPnsldXHn7F5GkFXW4m1ldM6sfvA/8DvgcmAGcH1jtfODlihayyqhZu/LDvUbgV/S7sX6ahBcu8CN44mHdx1C4Oz7bFpFKVZGW+4HAO2b2CfAh8B/n3OvAXcApZvYVcErgcWqomVG54e4CLXeAuk3g9+NhWx48Oxjyf4ntvrZvgJze8NRZsd2uiCRE1BfrcM6tArqEWb4B6FORQlVZ6bUrd0qCokKokVb8uFV3+P3jMOUPMHkoDHkOatWJzb52bfG337wdm+2JSELpDNVI1MyEgh2Vtz9XCDVKfP4e2Q8GPgKr5vmA3x2j8hQVFN/f9G1stikiCaNwj0R6JhTsqrz9FRUVd8uEyh5aHPDPDYlNwIf2teuiJSJJT+EeiZqZsWspl0dRwd7dMqH2CvgYtOCLQsL92/cqti0RSTiFeyRqVvJQSFcItp9fUfZQGPgwrJoLz5wDO36Ofl+FgW6ZGjVh2auxP2ArIpVK4R6JWnVh9y8w9x+Vs7+iMH3uJWWfB4Meg2/fhyd+BxtXR7evwnx/e8wl8MtP8M070W1HRKoEhXskDuzgb+f+vXLmmHGFpXfLhOp8jr+K1PY8ePxkWPth5PsKdssceiJkNIBPnot8GyJSZSjcI/HrU4vvP9AV1i2J376cA1fKAdVw2vSEC2dBZgOY2A8+f9Gf7LR2YfleHzygmlEPfnMBfDEdfv4mioKLSFWgcI9EzQy4IRc6/t4/fuIUWD0/PvsqKvS35Wm5B2X9ygf8wV3hhZFw92HwxMn+zNMy9xfsc0+HHpf6D5X3Hoy83CJSJSjcI5VR3/dxD3zU91NPOhMWT4z9fOsuEO77O6AaTt0mvovm2NHQsLVfltO77K6aYMs9rSY0OBi6DIGPJkXfh1+Wgl3w3Hn+WIGIxJzCPRo10vxIlYvnQeND4ZUx/qzRWAbhnpZ7FCcRp2dC37Fw5Wdw2El+2ROnwEdP7Wd/wXCv5W9PvNG34qf/qbgssbRqHqz4D8y8MfbbTjTn4MWL4cuZiS5JanhuKMy/O9GlSDoK94o4OBsu+9C3kr+cCQ8fC7P/BtvCToQZGRdFt0w4Q6fASTf7+zNGw7SLwp+IFWy510j3tw0OhjPu8WPe3/1nxcoQzqbAVa1cUey3nWiuCD6dAs+e609Ek4pZ8RrMuTPRpUg6CveKSkv3reRL3oZf9YG374H7OsDLo+HH5dFvN9haLu8B1dLUrAW9roarlkPzbPhsqi/fiv/u3ZUU2i0T1GUIdDgLZt8OiyZUrBwlBYdepuJB29CpHL5S6z1mdlfiOSYpQOEeK82OhCHPwOWLoetw+Ox5ePgYmHA6fDo18j/MaA6o7k+D5vDHef6s1lr1/LQFTw/yV5eC4m6ZYMsdwAzOyoF2v4NXr4RPJsemLFD87WHHz/DRk7HbblUQGu5z79L1b2Nl9bxElyCpKNxjrclh0O9euHIp9LkVtnwHL14E9x4Br10D331Uvn/2aA+oliX7PLj0Xehziz9R6b6O8OatxVeYSkvfe/2ateDcp6BtL5h+qR9iGQuhc9mkWrgH39uBnWD9ElgaozqrroL/A8teSWw5kozCPV7qZsHxV8HlH/vRK4f2hsWT4LET4aFj4O17i1vN4VTkgGpZatWF4/8Cf3ofug6Dd++HWbf650qGO/gDtEOfg1bHwLT/gQ8fq3gZCnf5LqfsYZC70B9gTRXB313XYT7g37i5ci/PmGqCXZMr/ls8TYaUSeEebzVq+GA/ZyJc/SX0ux/qNPb92Pd1hEn9YclzsGvb3q+L1QHV/WlyGAx4yI+Nb9Ye0utAet3w69aqC8Oe9100r10NMy6v2AW7C/P9eQPBE8Nm3Rb9tqqaYLdMWi0485+w7Qd49c/x7Z6Jx4imqqKoALIO99NirNG0GOWlcK9MtQ+AbiNh1OtwxcfQ+3o/amT6JTCunR8+t3KO/0eN1QHV8mjVHS59D65Z6bthSpNR3x9XOO4qP6zy3yf4bqZoFO723xKOPBPaHA8/LPXvPRUUhUzC1vI3fljp0pdgyTPx2d+6JfC3pn5uoVRTVAQ4OPw03/D47Pn473PVPHjp0qQ/VqJwT5TGh/pwv2IJjJoJnQfDl6/7y9zd1wHmBSYni2fLPZRZ+a7qVCMNTr7VdzXt/sWPn3/zlshb8QW7fMvWDM6ZBPWa+fcer5OmKlNouAP0/LP/AHvt2oqNoCrNz9/4b3prP4At62K//UQKHujPqO/nUPp0avyvhvbM2fDJs36upiSmcE80M2jdA868H/7ypQ+65tl+nDT4S/tVRYee4A/Mdhnix8E/eDR8MWPv1o5zpbd+CndDWoa/X7eJb90CvHQJ7Nwc37LHW8lwr5EGg3J819bTg2DT2hLrF/qpHqL9FhR63sKqudFtIxLffw6fvRD//UBIF1c69BzjHy94KL77zGjgb3/6Kr77iTOFe1WSngkdBsJ5k/249HOfhHZ9E12q0tVu5PvsR82EzIYwdTg8dhJ8PdsH1rOD/cHjcAcTC/P3PnibfR78/gn4bpGf+Kwyr1Uba3sCKeRgeIODYfiL/tjKUwP3Dvjlr8IbN8Gcv0W3v8KQcJ97V/zr7sWLYdqF5Z+UriJCPygbHwodBsGi8fH9hhdsUG1eu//1qjiFe1VVrym0H+ADv6pr3QP+OB/6P+i/yj49CO5t70/g+WkFvHkzbN+w92sKdxVPdRDU6Wx/4HnjKsg5MXnnnSnZcg86qBMMmwrb8uDxPsXz/QSnKfilRB2VV7DlfvYEf/3bt++Nbjvl1bCFv506omIXiCmPkqPG+tzs708etu8ghFjYnFsc6kk+wknhLrGRVhOOGu5P4jrjHjioI5x4ExzQ2o9jv/tQ+M9firtpCneHP3h75Jlwwat+exNO89M5hI6Jj9bsO2D8aZXT5VNauIP/ILxwph8pNP5U+O91vuUOsP4TePWqyKcsCF4d7Fcn+wbBh/+GhY9HX/6y1Aw0OLb94MsfT3vqMnDsqVEbOHs85C2Dl/8U+4OeP68pvv/6dfDLxthuvxIp3CW2amZA9/+BP0yDE66FIc8Wdy0tfBxuP8DPorlj074t96CDu/pJ2Q4/Hd4eBw/3gOWvVWyelo+f9vPkPHYSfP9Z9Nspj/2FO/izmS95xx9E/+Df/uzlDmf55xY9Ac+P2Pebzv4Ew71mJpz2D2jRzX+QxnrKiD372+V/Rydc648N/edqPwY9d3Hs97VnzqOQuvxVHzj5dvjiZd8NFUsFJa5FnMQnTincJb4O6gTnTfFhfWhvv+yVMT5og/PLhFOnsR92eW5gJsvJQ+GBLn7unmgmZqvf3A+l27UNJp4BG1ZGvo3yKizR2gwnsyGc9Qhc+TmM+QQ6nVP83LJXYOLp5e9XLsgHzB/DqH+Qr+9DevopI2beFHg+hgp2+IPhx1/tL8u48DE/ncXjJ/kup1gq7YPyt5dDl6Ew7y4/CilW4/xLXmj+lSuStntG4S7xZ+Zn0BzxMly/FgY/7f85j7uy7Ne27w+XLvD9yQcc4rtX7j0Spp7vxyOX92v5rq1w+Kn+HAMMHj0e3vtXbLp8Sgq98ElZGrb08/4c1sfPLtqorV+etxweyPbdUuFm8QxVsNN/YzLzj+tmwdDJ8JvzYcGD8OhxsT0DuGCX319aTf9N4cI3oXdgtNO8f8Q2DEurSzN/MP/Y0b4b6plz9u5SiVbwwvDBayEA/OMQ2Lml4tuuZAp3qVyZDXy/+u/uLL6iVVlq1oKOg3xf/OhFvrW4eh482d9f7vCNm+HbD/bfetu11Y+VbtzWT6DW5jh446/w717+gGYs+27L6pYJJzgH/58WwPCXipe/PQ7+frBvgf/0VfhyBsM2VGYDf3bskOdg5yZfV4+fAp9Pq/jsigU7i/vdAVodDb2vgxa/8a34R3rGbsTO/ibQq5Hm66zfffDtAnjoaN9NU5Ezp3cHXnvhG3v//r55O/ptJojCXZJLVjv/D33Vcj9jZaM28P7DMP538H9t/UVTFj4OeV/u3Uefv83Phgn+NedN8ccD8rf7eddzevvgC7bcQhXk+w+A8oZiNOEelF7bX2Dlts3+rOE+t/oPogUPwoPd4F9H+fl9ljwLP33tw75w195hG+qI02HMp3DqP/xIphdGwbhf+ympV78d3XGMcB8m4L9d9boGtuTCPYfDvLsr3oovT112GwWjF/pjNHP/F+7vDO/c768hHKlgt0x6bbh2dfH0GJPP89+ikkgcZqUSqQTpmdBlsP/ZsQm+ngWr3oKVc4sPgmU09N1Bzbv4s2mDJ6eA/1p/xBl+rpxPp8D8cT740uv6U907DvKjT9JqwUt/9DM79hwDp9xRdtliNenbgR38z/FX+eGhX73pzyH46o3i0/AbtPQBmBYmbIPSM6HHJXD0Rf6av59O9dMhfPwU1Gni++fb9vK3TQ8v+6zoki33oEaHwEl/hYM6++Gvb90J7z0AvW+AX/f1cxlFqrwflA1bwjkT/PV/3/q7nwhvzp1+v50H+w/MjHpl7y/44V6rrj+GMXSyPy9h1Vz/LWrDV75RkQRDlM1VgfkTunXr5hYtWpToYkgqcM4H4Zr3YN1H8N1iPzrGFfmTwtoPCP+6okJY866f0viLl2HHRt/PW7sRbA85gFu/uR+pcXC2D8Jwlv/Ht/QunufXi7XdO/x49tXzfZlzF0GLo/z7K6/8X/wVjr6e7bscgmO70+tC887+LOkDO/hvSlm/9ge4g+45wn/wDdjPBdSD23//EX9iGkDbE3xLuG0vP2KoPFNrfLfYj3AaOsUfMymvH77wc/l8OsV/Y0mr5b8BtTkOWvXw9RXu7O83bvZlvqVEq/+DHPjvNcWPf3Wy7/Zq2LL8ZYoDM1vsnOsW9jmFu6S83Tv9HCUZ9cu3fuFufwByzTv+TNJDfutbuM+fX7yOpfmZNH91EjQ+zLd662b5UTDLZvgTfC59zwdkVeecn8BuzXt+ErL1S/wH4u6QLqo6TXzINz7Ud191HQ5njCvf9jes9HXyQQ5sDcx9k17Hj6Rq3sV/kDTv7LvN8pb7fbc7xQ+3XPuhn79o2DRod3Lk761wt++P/3Km/+bz0wq/vEY6HNgemh4JzY7wt2np/uzbrHaBA+8l5G/303bPvKF42UGd/beDtr2gZfdKny5E4S5SUc757hAMPn7SH9zcuRm2rt97vYatYfO3/v5lH5beuq/qigp94P/0Nfz0pe+OyPvST1K27Xs49S445o+Rb3fzd/6bQvBDZP2nxQcxQ1kN6PEnP1/R5m+Lr4lQUb9s9BOsffu+/xD5cVnxBw74rruR//Un4ZWmqNCX/5Pn/DenvOX+m6Gl+Q+Ggzr5D/4DWvuWfcNWfohqHCYBVLiLxMuGlb6b5NsFvmW37qPicfjDXihfP2+yKSqMXVAVFfo6/OFzX39NDvPfhKZf4qeArtME6jbzJ8UFpz2ItR2b/AfY7l/gwI7+G1gkdm6GNQuKuwC//8wfVA5Vo6afX6hhq0Dgt/Tvq26W/0Bo3iWqoivcRSS5OOdHu9TNKh6/n0x2bQvMU5Prv3kE729a62+3fFd8QZ6Ov/dTKkRhf+Gu0TIiUvWY+cnzklVGPd+X3+yI8M8XFflJ17bnhb+0ZQzEbZy7mZ1qZivM7Gszuz5e+xERSTo1avjrGDQ7IrohouXZRTw2amZpwEPAaUB7YKiZtY/HvgoKi9hdWIEJpUREUlC8umWOBr52zq0CMLPJwADgi7Brr1gBvXtHvJPF9Q5m1BGD2JqWQUZRARlFhaS7QiLpoTMiO+ZgEawe8bYjWjd+x0osguMwkfaGJuPvJtL36ACH4SxwG1wWpu84tK5Dn937fsg6LvzynTVqsjtwvV3DYcFbBzX2lMLvzwLbNxw1qsAxt1Sx72/Xlfp86O+996bV/PXbuTEvT7zCvQUQehmTXOCY0BXM7GLgYoDOGfs5u24/Dv8ljxM2rab1zs3srFGT/Bppe/7AyyPSP+tw/5wx23YEERLJtiMpc6TbjjT2Iip3pNuOYPW4loPi4DR8GFsgXEtuqbR/fbfXOiHLjbDrANQqKqSWK9yzfvDDJPhBU2R74n2vD58iLK4Nheqi5G+3ZI3u7/+wef7WOJQofuEe7p3s9X6dczlADvjRMsydG/FO6gEPRFE4EZGUsJ8PjXgdUM0FWoU8bgmk2GXZRUSqrniF+0KgnZm1NbNawBBgRpz2JSIiJcSlW8Y5V2Bmo4GZQBow3jm3NB77EhGRfcXtJCbn3GvAa/HavoiIlE4X6xARSUEKdxGRFKRwFxFJQQp3EZEUVCWm/DWzPGBNBTaRBURxNdyUpjrZl+pkX6qT8JKlXg5xzoWdPrNKhHtFmdmi0uY0rq5UJ/tSnexLdRJeKtSLumVERFKQwl1EJAWlSrjnJLoAVZDqZF+qk32pTsJL+npJiT53ERHZW6q03EVEJITCXUQkBSV1uFfXi3CbWSsze8vMlpnZUjMbE1je2MzeNLOvAreNQl5zQ6CeVphZ38SVPr7MLM3MPjazVwOPq3WdmNkBZvaCmS0P/L0cW93rBMDMrgz873xuZs+ZWWbK1YtzLil/8FMJrwQOBWoBnwDtE12uSnrvzYGjAvfrA1/iL0T+f8D1geXXA/8I3G8fqJ8MoG2g3tIS/T7iVDdXAc8CrwYeV+s6ASYB/xO4Xws4QHVCC2A1UDvweCpwQarVSzK33PdchNs5lw8EL8Kd8pxz651zHwXubwWW4f9gB+D/mQncDgzcHwBMds7tcs6tBr7G119KMbOWwBnA4yGLq22dmFkDoBfwBIBzLt85t4lqXCchagK1zawmUAd/pbiUqpdkDvdwF+FukaCyJIyZtQG6Ah8ABzrn1oP/AACaBVarLnV1P3AtUBSyrDrXyaFAHjAh0FX1uJnVpXrXCc6574BxwLfAemCzc+4NUqxekjncy7wId6ozs3rANODPzrkt+1s1zLKUqisz6wf86JxbXN6XhFmWUnWCb50eBTzinOsKbMd3N5SmOtQJgb70AfguloOBumb2h/29JMyyKl8vyRzu1foi3GaWjg/2Z5xzLwYW/2BmzQPPNwd+DCyvDnXVE+hvZt/gu+hOMrOnqd51kgvkOuc+CDx+AR/21blOAE4GVjvn8pxzu4EXgd+SYvWSzOFebS/CbWaG70dd5py7N+SpGcD5gfvnAy+HLB9iZhlm1hZoB3xYWeWtDM65G5xzLZ1zbfB/C3Occ3+getfJ98BaMzs8sKgP8AXVuE4CvgV6mFmdwP9SH/xxq5Sql7hdQzXeXPW+CHdPYDjwmZktCSy7EbgLmGpmF+L/gM8BcM4tNbOp+H/sAuAy51xhpZc6Map7nVwOPBNoAK0CRuIbddW2TpxzH5jZC8BH+Pf5MX66gXqkUL1o+gERkRSUzN0yIiJSCoW7iEgKUriLiKQghbuISApSuIuIpCCFu4hIClK4i4ikoP8HxIU+9636fawAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (500, 500) to (512, 512) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n",
      "[swscaler @ 0x571ba80] Warning: data is not aligned! This can lead to a speed loss\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Video saved to test_video.mp4.\n"
     ]
    }
   ],
   "source": [
    "instance_PPO_Agent.explore(5)\n",
    "plt.plot(instance_PPO_Agent.history_training[2])\n",
    "plt.show()\n",
    "plt.axhline(0,color='r')\n",
    "plt.plot(instance_PPO_Agent.history_training[0][0:],label='actor_loss')\n",
    "plt.plot(instance_PPO_Agent.history_training[1][0:],label='critic_loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "#instance_PPO_Agent.test()\n",
    "video = []\n",
    "state = instance_PPO_Agent.instance_environment.reset()\n",
    "episodic_reward = 0\n",
    "while True:\n",
    "    state = state.reshape(1,-1)\n",
    "    action,action_distribution = instance_PPO_Agent.get_action_and_distribution(state)\n",
    "    state_next,reward,done,info = instance_PPO_Agent.instance_environment.step(action)\n",
    "    reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']\n",
    "    state = state_next\n",
    "    video.append(instance_PPO_Agent.instance_environment.render(mode ='rgb_array'))\n",
    "    if done:\n",
    "        break\n",
    "writer = imageio.get_writer('test_video.mp4',fps=30) \n",
    "for image in video:\n",
    "    writer.append_data(np.uint8(image))\n",
    "writer.close()\n",
    "print('Video saved to test_video.mp4.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (500, 500) to (512, 512) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n",
      "[swscaler @ 0x6762a80] Warning: data is not aligned! This can lead to a speed loss\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Video saved to test_video.mp4.\n"
     ]
    }
   ],
   "source": [
    "import imageio\n",
    "video = []\n",
    "state = instance_PPO_Agent.instance_environment.reset()\n",
    "episodic_reward = 0\n",
    "while True:\n",
    "    state = state.reshape(1,-1)\n",
    "    action,action_distribution = instance_PPO_Agent.get_action_and_distribution(state)\n",
    "    state_next,reward,done,info = instance_PPO_Agent.instance_environment.step(action)\n",
    "    reward = info['reward_linvel']+info['reward_quadctrl']+info['reward_impact']\n",
    "    state = state_next\n",
    "    video.append(instance_PPO_Agent.instance_environment.render(mode ='rgb_array'))\n",
    "    if done:\n",
    "        break\n",
    "writer = imageio.get_writer('test_video.mp4',fps=30) \n",
    "for image in video:\n",
    "    writer.append_data(np.uint8(image))\n",
    "writer.close()\n",
    "print('Video saved to test_video.mp4.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1, 17), dtype=float32, numpy=\n",
       " array([[ 0.11227124, -0.3174024 , -0.31886122,  0.24518172,  0.11826874,\n",
       "          0.0990658 ,  0.17539349,  0.1228866 , -0.31950098,  0.24559827,\n",
       "         -0.0161692 , -0.3129159 ,  0.3086023 ,  0.2985952 , -0.31999817,\n",
       "          0.27059862,  0.30623218]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(1, 17), dtype=float32, numpy=\n",
       " array([[0.05      , 0.05      , 0.05      , 0.05      , 0.05      ,\n",
       "         0.05      , 0.05      , 0.05      , 0.05      , 0.05      ,\n",
       "         0.05      , 0.05      , 0.05000001, 0.05      , 0.05      ,\n",
       "         0.05      , 0.05      ]], dtype=float32)>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "instance_PPO_Agent.instance_PPO_Policy_Model(state.reshape(1,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1, 17), dtype=float32, numpy=\n",
       " array([[-0.3199999 ,  0.31999964,  0.14556305,  0.31922042,  0.31995192,\n",
       "         -0.31992975,  0.31995857,  0.32000002, -0.32000002, -0.20125969,\n",
       "          0.32000002, -0.2806096 , -0.31453428,  0.03673209,  0.3199769 ,\n",
       "         -0.32000002,  0.18682738]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(1, 17), dtype=float32, numpy=\n",
       " array([[0.05119522, 0.05029236, 0.05180176, 0.05082433, 0.05920524,\n",
       "         0.05000708, 0.08999993, 0.05019069, 0.07866695, 0.05000003,\n",
       "         0.051122  , 0.05012888, 0.05000375, 0.05      , 0.05000001,\n",
       "         0.08729389, 0.05000007]], dtype=float32)>]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "instance_PPO_Agent.instance_PPO_Policy_Model(state.reshape(1,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.3011645 ,  0.2361007 ,  0.36315414, -0.39324334, -0.16203973,\n",
       "         0.43837494, -0.34902033,  0.31892747, -0.32258826,  0.29584417,\n",
       "         0.30814978,  0.18339801, -0.3093988 ,  0.32704595,  0.36235338,\n",
       "         0.5186783 ,  0.2600025 ]], dtype=float32)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.36367834, -0.297418  , -0.17439544,  0.22681314,  0.33496067,\n",
       "         0.365591  ,  0.26825655, -0.00988701, -0.2982839 , -0.328961  ,\n",
       "         0.209575  ,  0.30764115,  0.38200277,  0.3068128 ,  0.33857086,\n",
       "         0.29352102,  0.15110895]], dtype=float32)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import gym\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义神经网络模型\n",
    "def create_model(input_shape, output_shape):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Dense(64, activation='relu')(inputs)\n",
    "    x = Dense(64, activation='relu')(x)\n",
    "    outputs = Dense(output_shape, activation='tanh')(x)\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    model.compile(optimizer=Adam(learning_rate=1e-3), loss='mse')\n",
    "    return model\n",
    "# 定义PPO算法\n",
    "def ppo_train(policy_model, value_model, states, actions, advantages, old_probs, epochs=10, batch_size=64):\n",
    "    for epoch in range(epochs):\n",
    "        indices = np.random.randint(0, states.shape[0], size=batch_size)\n",
    "        s = states[indices]\n",
    "        a = actions[indices]\n",
    "        adv = advantages[indices]\n",
    "        op = old_probs[indices]\n",
    "\n",
    "        # 更新策略网络\n",
    "        with tf.GradientTape() as tape:\n",
    "            new_probs = policy_model(s)\n",
    "            prob_ratio = tf.exp(new_probs - op)\n",
    "            surr = prob_ratio * adv\n",
    "            clip_surr = tf.clip_by_value(prob_ratio, 1 - 0.2, 1 + 0.2) * adv\n",
    "            policy_loss = -tf.reduce_mean(tf.minimum(surr, clip_surr))\n",
    "\n",
    "        policy_grads = tape.gradient(policy_loss, policy_model.trainable_variables)\n",
    "        policy_model.optimizer.apply_gradients(zip(policy_grads, policy_model.trainable_variables))\n",
    "\n",
    "        # 更新价值网络\n",
    "        with tf.GradientTape() as tape:\n",
    "            v_pred = value_model(s)\n",
    "            value_loss = tf.reduce_mean(tf.square(v_pred - adv))\n",
    "\n",
    "        value_grads = tape.gradient(value_loss, value_model.trainable_variables)\n",
    "        value_model.optimizer.apply_gradients(zip(value_grads, value_model.trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建gym环境\n",
    "env = gym.make('Humanoid-v2')\n",
    "# 创建策略和价值网络\n",
    "state_shape = env.observation_space.shape\n",
    "action_shape = env.action_space.shape[0]\n",
    "policy_model = create_model(state_shape, action_shape)\n",
    "value_model = create_model(state_shape, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-05-17 16:15:04.491812: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-17 16:15:04.758714: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-17 16:15:04.759883: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-05-17 16:15:06.602094: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "/home/dell/anaconda3/envs/CX/lib/python3.8/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "2024-05-17 16:15:10.468981: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1960] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'action' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb Cell 22\u001b[0m in \u001b[0;36m<cell line: 58>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=71'>72</a>\u001b[0m     states\u001b[39m.\u001b[39mappend(state)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=72'>73</a>\u001b[0m     \u001b[39m# actions.append(action)\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=73'>74</a>\u001b[0m     \u001b[39m# rewards.append(reward)\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=74'>75</a>\u001b[0m     \u001b[39m# dones.append(done)\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=76'>77</a>\u001b[0m     state, reward, done, _ \u001b[39m=\u001b[39m env\u001b[39m.\u001b[39mstep(action)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=78'>79</a>\u001b[0m \u001b[39m# 计算优势函数和旧的概率\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=79'>80</a>\u001b[0m \u001b[39m# ...\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=80'>81</a>\u001b[0m \n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=81'>82</a>\u001b[0m \u001b[39m# 使用PPO算法进行训练\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/dell/CX/cx_project/neuron-vis/project_robot/robot_ppo.ipynb#X30sZmlsZQ%3D%3D?line=82'>83</a>\u001b[0m ppo_train(policy_model, value_model, np\u001b[39m.\u001b[39marray(states), np\u001b[39m.\u001b[39marray(actions), advantages, old_probs)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'action' is not defined"
     ]
    }
   ],
   "source": [
    "# 训练循环\n",
    "episodes = 1000\n",
    "for episode in range(episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    states = []\n",
    "    actions = []\n",
    "    rewards = []\n",
    "    dones = []\n",
    "\n",
    "    while not done:\n",
    "        # 在这里添加你的代码来选择和执行动作\n",
    "        # ...\n",
    "\n",
    "        # 记录经验\n",
    "        states.append(state)\n",
    "        # actions.append(action)\n",
    "        # rewards.append(reward)\n",
    "        # dones.append(done)\n",
    "\n",
    "        state, reward, done, _ = env.step(action)\n",
    "\n",
    "    # 计算优势函数和旧的概率\n",
    "    # ...\n",
    "\n",
    "    # 使用PPO算法进行训练\n",
    "    ppo_train(policy_model, value_model, np.array(states), np.array(actions), advantages, old_probs)\n",
    "\n",
    "    print(f\"Episode: {episode}, Total Reward: {total_reward}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('CX')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6684bfb8f990107dc2e41da7bb6829210ca36b987584a9d9d8b153337675e487"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
